]> git.ipfire.org Git - thirdparty/glibc.git/blame - scripts/build-many-glibcs.py
projects / thirdparty / glibc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update copyright dates with scripts/update-copyrights.
[thirdparty/glibc.git] / scripts / build-many-glibcs.py
CommitLineData
14f95a42
JM		 1#!/usr/bin/python3
2# Build many configurations of glibc.
d614a753 3# Copyright (C) 2016-2020 Free Software Foundation, Inc.
14f95a42
JM		 4# This file is part of the GNU C Library.
5#
6# The GNU C Library is free software; you can redistribute it and/or
7# modify it under the terms of the GNU Lesser General Public
8# License as published by the Free Software Foundation; either
9# version 2.1 of the License, or (at your option) any later version.
10#
11# The GNU C Library is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14# Lesser General Public License for more details.
15#
16# You should have received a copy of the GNU Lesser General Public
17# License along with the GNU C Library; if not, see
5a82c748 18# <https://www.gnu.org/licenses/>.
14f95a42
JM		 19
20"""Build many configurations of glibc.
21
22This script takes as arguments a directory name (containing a src
23subdirectory with sources of the relevant toolchain components) and a
24description of what to do: 'checkout', to check out sources into that
4d602bce 25directory, 'bot-cycle', to run a series of checkout and build steps,
a1f6a9ab
JM		 26'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27libraries required by the toolchain, 'compilers', to build
28cross-compilers for various configurations, or 'glibcs', to build
29glibc for various configurations and run the compilation parts of the
30testsuite. Subsequent arguments name the versions of components to
31check out (<component>-<version), for 'checkout', or, for actions
32other than 'checkout' and 'bot-cycle', name configurations for which
33compilers or glibc are to be built.
4d602bce 34
14f95a42
JM		 35"""
36
37import argparse
bf469f0c 38import datetime
4d602bce
JM		 39import email.mime.text
40import email.utils
02c78f02 41import json
14f95a42 42import os
14f95a42
JM		 43import re
44import shutil
4d602bce 45import smtplib
14f95a42
JM		 46import stat
47import subprocess
48import sys
a1f6a9ab 49import time
14f95a42
JM		 50import urllib.request
51
64235ccc
SN		 52try:
53 subprocess.run
54except:
55 class _CompletedProcess:
56 def __init__(self, args, returncode, stdout=None, stderr=None):
57 self.args = args
58 self.returncode = returncode
59 self.stdout = stdout
60 self.stderr = stderr
61
62 def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
63 assert(timeout is None)
64 with subprocess.Popen(*popenargs, **kwargs) as process:
65 try:
66 stdout, stderr = process.communicate(input)
67 except:
68 process.kill()
69 process.wait()
70 raise
71 returncode = process.poll()
72 if check and returncode:
73 raise subprocess.CalledProcessError(returncode, popenargs)
74 return _CompletedProcess(popenargs, returncode, stdout, stderr)
75
76 subprocess.run = _run
77
14f95a42 78
0c95f51d 79class Context(object):
14f95a42
JM		 80 """The global state associated with builds in a given directory."""
81
297635d8 82 def __init__(self, topdir, parallelism, keep, replace_sources, strip,
b1176270 83 full_gcc, action):
14f95a42
JM		 84 """Initialize the context."""
85 self.topdir = topdir
86 self.parallelism = parallelism
87 self.keep = keep
02c78f02 88 self.replace_sources = replace_sources
297635d8 89 self.strip = strip
b1176270 90 self.full_gcc = full_gcc
14f95a42 91 self.srcdir = os.path.join(topdir, 'src')
02c78f02 92 self.versions_json = os.path.join(self.srcdir, 'versions.json')
bf469f0c 93 self.build_state_json = os.path.join(topdir, 'build-state.json')
4d602bce 94 self.bot_config_json = os.path.join(topdir, 'bot-config.json')
14f95a42
JM		 95 self.installdir = os.path.join(topdir, 'install')
96 self.host_libraries_installdir = os.path.join(self.installdir,
97 'host-libraries')
98 self.builddir = os.path.join(topdir, 'build')
99 self.logsdir = os.path.join(topdir, 'logs')
a1f6a9ab 100 self.logsdir_old = os.path.join(topdir, 'logs-old')
14f95a42
JM		 101 self.makefile = os.path.join(self.builddir, 'Makefile')
102 self.wrapper = os.path.join(self.builddir, 'wrapper')
103 self.save_logs = os.path.join(self.builddir, 'save-logs')
a1c9859b 104 self.script_text = self.get_script_text()
14f95a42
JM		 105 if action != 'checkout':
106 self.build_triplet = self.get_build_triplet()
107 self.glibc_version = self.get_glibc_version()
108 self.configs = {}
109 self.glibc_configs = {}
110 self.makefile_pieces = ['.PHONY: all\n']
111 self.add_all_configs()
02c78f02 112 self.load_versions_json()
bf469f0c
JM		 113 self.load_build_state_json()
114 self.status_log_list = []
f0166c16 115 self.email_warning = False
14f95a42 116
a1c9859b
JM		 117 def get_script_text(self):
118 """Return the text of this script."""
119 with open(sys.argv[0], 'r') as f:
120 return f.read()
121
122 def exec_self(self):
123 """Re-execute this script with the same arguments."""
36820ce9 124 sys.stdout.flush()
a1c9859b
JM		 125 os.execv(sys.executable, [sys.executable] + sys.argv)
126
14f95a42
JM		 127 def get_build_triplet(self):
128 """Determine the build triplet with config.guess."""
129 config_guess = os.path.join(self.component_srcdir('gcc'),
130 'config.guess')
131 cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
132 check=True, universal_newlines=True).stdout
133 return cg_out.rstrip()
134
135 def get_glibc_version(self):
136 """Determine the glibc version number (major.minor)."""
137 version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
138 with open(version_h, 'r') as f:
139 lines = f.readlines()
140 starttext = '#define VERSION "'
141 for l in lines:
142 if l.startswith(starttext):
143 l = l[len(starttext):]
144 l = l.rstrip('"\n')
145 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
146 return '%s.%s' % m.group(1, 2)
147 print('error: could not determine glibc version')
148 exit(1)
149
150 def add_all_configs(self):
151 """Add all known glibc build configurations."""
152 self.add_config(arch='aarch64',
6cb86fd2
SN		 153 os_name='linux-gnu',
154 extra_glibcs=[{'variant': 'disable-multi-arch',
155 'cfg': ['--disable-multi-arch']}])
14f95a42
JM		 156 self.add_config(arch='aarch64_be',
157 os_name='linux-gnu')
158 self.add_config(arch='alpha',
159 os_name='linux-gnu')
160 self.add_config(arch='arm',
72edea80
FW		 161 os_name='linux-gnueabi',
162 extra_glibcs=[{'variant': 'v4t',
163 'ccopts': '-march=armv4t'}])
14f95a42
JM		 164 self.add_config(arch='armeb',
165 os_name='linux-gnueabi')
166 self.add_config(arch='armeb',
167 os_name='linux-gnueabi',
168 variant='be8',
169 gcc_cfg=['--with-arch=armv7-a'])
170 self.add_config(arch='arm',
3d265601 171 os_name='linux-gnueabihf',
8e52f573 172 gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
3d265601 173 extra_glibcs=[{'variant': 'v7a',
8e52f573 174 'ccopts': '-march=armv7-a -mfpu=vfpv3'},
3d265601 175 {'variant': 'v7a-disable-multi-arch',
8e52f573 176 'ccopts': '-march=armv7-a -mfpu=vfpv3',
3d265601 177 'cfg': ['--disable-multi-arch']}])
14f95a42 178 self.add_config(arch='armeb',
5170fa49 179 os_name='linux-gnueabihf',
8e52f573 180 gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
14f95a42
JM		 181 self.add_config(arch='armeb',
182 os_name='linux-gnueabihf',
183 variant='be8',
8e52f573
JM		 184 gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
185 '--with-fpu=vfpv3'])
5f72b005
MH		 186 self.add_config(arch='csky',
187 os_name='linux-gnuabiv2',
188 variant='soft',
189 gcc_cfg=['--disable-multilib'])
190 self.add_config(arch='csky',
191 os_name='linux-gnuabiv2',
192 gcc_cfg=['--with-float=hard', '--disable-multilib'])
14f95a42
JM		 193 self.add_config(arch='hppa',
194 os_name='linux-gnu')
394b5bac
JM		 195 self.add_config(arch='i686',
196 os_name='gnu')
14f95a42
JM		 197 self.add_config(arch='ia64',
198 os_name='linux-gnu',
199 first_gcc_cfg=['--with-system-libunwind'])
200 self.add_config(arch='m68k',
201 os_name='linux-gnu',
202 gcc_cfg=['--disable-multilib'])
203 self.add_config(arch='m68k',
204 os_name='linux-gnu',
205 variant='coldfire',
206 gcc_cfg=['--with-arch=cf', '--disable-multilib'])
3005b705
JM		 207 self.add_config(arch='m68k',
208 os_name='linux-gnu',
209 variant='coldfire-soft',
210 gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
211 '--disable-multilib'])
14f95a42
JM		 212 self.add_config(arch='microblaze',
213 os_name='linux-gnu',
214 gcc_cfg=['--disable-multilib'])
215 self.add_config(arch='microblazeel',
216 os_name='linux-gnu',
217 gcc_cfg=['--disable-multilib'])
218 self.add_config(arch='mips64',
219 os_name='linux-gnu',
220 gcc_cfg=['--with-mips-plt'],
221 glibcs=[{'variant': 'n32'},
222 {'arch': 'mips',
223 'ccopts': '-mabi=32'},
224 {'variant': 'n64',
225 'ccopts': '-mabi=64'}])
226 self.add_config(arch='mips64',
227 os_name='linux-gnu',
228 variant='soft',
229 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
8df5d347 230 glibcs=[{'variant': 'n32-soft'},
14f95a42
JM		 231 {'variant': 'soft',
232 'arch': 'mips',
8df5d347 233 'ccopts': '-mabi=32'},
14f95a42 234 {'variant': 'n64-soft',
8df5d347 235 'ccopts': '-mabi=64'}])
14f95a42
JM		 236 self.add_config(arch='mips64',
237 os_name='linux-gnu',
238 variant='nan2008',
239 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
240 '--with-arch-64=mips64r2',
241 '--with-arch-32=mips32r2'],
242 glibcs=[{'variant': 'n32-nan2008'},
243 {'variant': 'nan2008',
244 'arch': 'mips',
245 'ccopts': '-mabi=32'},
246 {'variant': 'n64-nan2008',
247 'ccopts': '-mabi=64'}])
248 self.add_config(arch='mips64',
249 os_name='linux-gnu',
250 variant='nan2008-soft',
251 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
252 '--with-arch-64=mips64r2',
253 '--with-arch-32=mips32r2',
254 '--with-float=soft'],
8df5d347 255 glibcs=[{'variant': 'n32-nan2008-soft'},
14f95a42
JM		 256 {'variant': 'nan2008-soft',
257 'arch': 'mips',
8df5d347 258 'ccopts': '-mabi=32'},
14f95a42 259 {'variant': 'n64-nan2008-soft',
8df5d347 260 'ccopts': '-mabi=64'}])
14f95a42
JM		 261 self.add_config(arch='mips64el',
262 os_name='linux-gnu',
263 gcc_cfg=['--with-mips-plt'],
264 glibcs=[{'variant': 'n32'},
265 {'arch': 'mipsel',
266 'ccopts': '-mabi=32'},
267 {'variant': 'n64',
268 'ccopts': '-mabi=64'}])
269 self.add_config(arch='mips64el',
270 os_name='linux-gnu',
271 variant='soft',
272 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
8df5d347 273 glibcs=[{'variant': 'n32-soft'},
14f95a42
JM		 274 {'variant': 'soft',
275 'arch': 'mipsel',
8df5d347 276 'ccopts': '-mabi=32'},
14f95a42 277 {'variant': 'n64-soft',
8df5d347 278 'ccopts': '-mabi=64'}])
14f95a42
JM		 279 self.add_config(arch='mips64el',
280 os_name='linux-gnu',
281 variant='nan2008',
282 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
283 '--with-arch-64=mips64r2',
284 '--with-arch-32=mips32r2'],
285 glibcs=[{'variant': 'n32-nan2008'},
286 {'variant': 'nan2008',
287 'arch': 'mipsel',
288 'ccopts': '-mabi=32'},
289 {'variant': 'n64-nan2008',
290 'ccopts': '-mabi=64'}])
291 self.add_config(arch='mips64el',
292 os_name='linux-gnu',
293 variant='nan2008-soft',
294 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
295 '--with-arch-64=mips64r2',
296 '--with-arch-32=mips32r2',
297 '--with-float=soft'],
8df5d347 298 glibcs=[{'variant': 'n32-nan2008-soft'},
14f95a42
JM		 299 {'variant': 'nan2008-soft',
300 'arch': 'mipsel',
8df5d347 301 'ccopts': '-mabi=32'},
14f95a42 302 {'variant': 'n64-nan2008-soft',
8df5d347 303 'ccopts': '-mabi=64'}])
a2e487ce
DM		 304 self.add_config(arch='mipsisa64r6el',
305 os_name='linux-gnu',
306 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
307 '--with-arch-64=mips64r6',
308 '--with-arch-32=mips32r6',
309 '--with-float=hard'],
310 glibcs=[{'variant': 'n32'},
311 {'arch': 'mipsisa32r6el',
312 'ccopts': '-mabi=32'},
313 {'variant': 'n64',
314 'ccopts': '-mabi=64'}])
14f95a42
JM		 315 self.add_config(arch='nios2',
316 os_name='linux-gnu')
317 self.add_config(arch='powerpc',
318 os_name='linux-gnu',
4179178b
JM		 319 gcc_cfg=['--disable-multilib', '--enable-secureplt'],
320 extra_glibcs=[{'variant': 'power4',
321 'ccopts': '-mcpu=power4',
322 'cfg': ['--with-cpu=power4']}])
14f95a42
JM		 323 self.add_config(arch='powerpc',
324 os_name='linux-gnu',
325 variant='soft',
326 gcc_cfg=['--disable-multilib', '--with-float=soft',
8df5d347 327 '--enable-secureplt'])
14f95a42
JM		 328 self.add_config(arch='powerpc64',
329 os_name='linux-gnu',
330 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
331 self.add_config(arch='powerpc64le',
332 os_name='linux-gnu',
333 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
0fd5d876
PD		 334 self.add_config(arch='riscv64',
335 os_name='linux-gnu',
336 variant='rv64imac-lp64',
337 gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
338 '--disable-multilib'])
339 self.add_config(arch='riscv64',
340 os_name='linux-gnu',
341 variant='rv64imafdc-lp64',
342 gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
343 '--disable-multilib'])
344 self.add_config(arch='riscv64',
345 os_name='linux-gnu',
346 variant='rv64imafdc-lp64d',
347 gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
348 '--disable-multilib'])
14f95a42
JM		 349 self.add_config(arch='s390x',
350 os_name='linux-gnu',
351 glibcs=[{},
352 {'arch': 's390', 'ccopts': '-m31'}])
14f95a42 353 self.add_config(arch='sh3',
c89721e2 354 os_name='linux-gnu')
14f95a42 355 self.add_config(arch='sh3eb',
c89721e2 356 os_name='linux-gnu')
14f95a42 357 self.add_config(arch='sh4',
c89721e2 358 os_name='linux-gnu')
14f95a42 359 self.add_config(arch='sh4eb',
c89721e2 360 os_name='linux-gnu')
14f95a42
JM		 361 self.add_config(arch='sh4',
362 os_name='linux-gnu',
363 variant='soft',
8df5d347 364 gcc_cfg=['--without-fp'])
14f95a42
JM		 365 self.add_config(arch='sh4eb',
366 os_name='linux-gnu',
367 variant='soft',
8df5d347 368 gcc_cfg=['--without-fp'])
14f95a42
JM		 369 self.add_config(arch='sparc64',
370 os_name='linux-gnu',
371 glibcs=[{},
372 {'arch': 'sparcv9',
5d9b7b9f 373 'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
066d80a7
AZ		 374 extra_glibcs=[{'variant': 'leon3',
375 'arch' : 'sparcv8',
376 'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
377 {'variant': 'disable-multi-arch',
0c097378
JM		 378 'cfg': ['--disable-multi-arch']},
379 {'variant': 'disable-multi-arch',
380 'arch': 'sparcv9',
066d80a7 381 'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
0c097378 382 'cfg': ['--disable-multi-arch']}])
14f95a42
JM		 383 self.add_config(arch='x86_64',
384 os_name='linux-gnu',
385 gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
386 glibcs=[{},
387 {'variant': 'x32', 'ccopts': '-mx32'},
388 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
389 extra_glibcs=[{'variant': 'disable-multi-arch',
390 'cfg': ['--disable-multi-arch']},
7244f13d
JM		 391 {'variant': 'enable-obsolete',
392 'cfg': ['--enable-obsolete-rpc',
393 '--enable-obsolete-nsl']},
1a49fc59
L		 394 {'variant': 'static-pie',
395 'cfg': ['--enable-static-pie']},
396 {'variant': 'x32-static-pie',
397 'ccopts': '-mx32',
398 'cfg': ['--enable-static-pie']},
399 {'variant': 'static-pie',
400 'arch': 'i686',
401 'ccopts': '-m32 -march=i686',
402 'cfg': ['--enable-static-pie']},
14f95a42
JM		 403 {'variant': 'disable-multi-arch',
404 'arch': 'i686',
405 'ccopts': '-m32 -march=i686',
406 'cfg': ['--disable-multi-arch']},
7244f13d
JM		 407 {'variant': 'enable-obsolete',
408 'arch': 'i686',
409 'ccopts': '-m32 -march=i686',
410 'cfg': ['--enable-obsolete-rpc',
411 '--enable-obsolete-nsl']},
14f95a42
JM		 412 {'arch': 'i486',
413 'ccopts': '-m32 -march=i486'},
414 {'arch': 'i586',
415 'ccopts': '-m32 -march=i586'}])
416
417 def add_config(self, **args):
418 """Add an individual build configuration."""
419 cfg = Config(self, **args)
420 if cfg.name in self.configs:
421 print('error: duplicate config %s' % cfg.name)
422 exit(1)
423 self.configs[cfg.name] = cfg
424 for c in cfg.all_glibcs:
425 if c.name in self.glibc_configs:
426 print('error: duplicate glibc config %s' % c.name)
427 exit(1)
428 self.glibc_configs[c.name] = c
429
430 def component_srcdir(self, component):
431 """Return the source directory for a given component, e.g. gcc."""
432 return os.path.join(self.srcdir, component)
433
434 def component_builddir(self, action, config, component, subconfig=None):
435 """Return the directory to use for a build."""
436 if config is None:
437 # Host libraries.
438 assert subconfig is None
439 return os.path.join(self.builddir, action, component)
440 if subconfig is None:
441 return os.path.join(self.builddir, action, config, component)
442 else:
443 # glibc build as part of compiler build.
444 return os.path.join(self.builddir, action, config, component,
445 subconfig)
446
447 def compiler_installdir(self, config):
448 """Return the directory in which to install a compiler."""
449 return os.path.join(self.installdir, 'compilers', config)
450
451 def compiler_bindir(self, config):
452 """Return the directory in which to find compiler binaries."""
453 return os.path.join(self.compiler_installdir(config), 'bin')
454
455 def compiler_sysroot(self, config):
456 """Return the sysroot directory for a compiler."""
457 return os.path.join(self.compiler_installdir(config), 'sysroot')
458
459 def glibc_installdir(self, config):
460 """Return the directory in which to install glibc."""
461 return os.path.join(self.installdir, 'glibcs', config)
462
463 def run_builds(self, action, configs):
464 """Run the requested builds."""
465 if action == 'checkout':
466 self.checkout(configs)
467 return
4d602bce
JM		 468 if action == 'bot-cycle':
469 if configs:
470 print('error: configurations specified for bot-cycle')
471 exit(1)
472 self.bot_cycle()
473 return
a1f6a9ab
JM		 474 if action == 'bot':
475 if configs:
476 print('error: configurations specified for bot')
477 exit(1)
478 self.bot()
479 return
bf469f0c
JM		 480 if action == 'host-libraries' and configs:
481 print('error: configurations specified for host-libraries')
482 exit(1)
483 self.clear_last_build_state(action)
484 build_time = datetime.datetime.utcnow()
485 if action == 'host-libraries':
486 build_components = ('gmp', 'mpfr', 'mpc')
487 old_components = ()
488 old_versions = {}
14f95a42
JM		 489 self.build_host_libraries()
490 elif action == 'compilers':
394b5bac
JM		 491 build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
492 'gnumach', 'hurd')
bf469f0c
JM		 493 old_components = ('gmp', 'mpfr', 'mpc')
494 old_versions = self.build_state['host-libraries']['build-versions']
14f95a42
JM		 495 self.build_compilers(configs)
496 else:
bf469f0c 497 build_components = ('glibc',)
394b5bac
JM		 498 old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
499 'mig', 'gnumach', 'hurd')
bf469f0c 500 old_versions = self.build_state['compilers']['build-versions']
14f95a42
JM		 501 self.build_glibcs(configs)
502 self.write_files()
503 self.do_build()
bf469f0c
JM		 504 if configs:
505 # Partial build, do not update stored state.
506 return
507 build_versions = {}
508 for k in build_components:
509 if k in self.versions:
510 build_versions[k] = {'version': self.versions[k]['version'],
511 'revision': self.versions[k]['revision']}
512 for k in old_components:
513 if k in old_versions:
514 build_versions[k] = {'version': old_versions[k]['version'],
515 'revision': old_versions[k]['revision']}
516 self.update_build_state(action, build_time, build_versions)
14f95a42
JM		 517
518 @staticmethod
519 def remove_dirs(*args):
520 """Remove directories and their contents if they exist."""
521 for dir in args:
522 shutil.rmtree(dir, ignore_errors=True)
523
524 @staticmethod
525 def remove_recreate_dirs(*args):
526 """Remove directories if they exist, and create them as empty."""
527 Context.remove_dirs(*args)
528 for dir in args:
529 os.makedirs(dir, exist_ok=True)
530
531 def add_makefile_cmdlist(self, target, cmdlist, logsdir):
532 """Add makefile text for a list of commands."""
533 commands = cmdlist.makefile_commands(self.wrapper, logsdir)
534 self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
535 (target, target, target, commands))
bf469f0c 536 self.status_log_list.extend(cmdlist.status_logs(logsdir))
14f95a42
JM		 537
538 def write_files(self):
539 """Write out the Makefile and wrapper script."""
540 mftext = ''.join(self.makefile_pieces)
541 with open(self.makefile, 'w') as f:
542 f.write(mftext)
543 wrapper_text = (
544 '#!/bin/sh\n'
545 'prev_base=$1\n'
546 'this_base=$2\n'
547 'desc=$3\n'
548 'dir=$4\n'
549 'path=$5\n'
550 'shift 5\n'
551 'prev_status=$prev_base-status.txt\n'
552 'this_status=$this_base-status.txt\n'
553 'this_log=$this_base-log.txt\n'
554 'date > "$this_log"\n'
555 'echo >> "$this_log"\n'
556 'echo "Description: $desc" >> "$this_log"\n'
8885f979
JM		 557 'printf "%s" "Command:" >> "$this_log"\n'
558 'for word in "$@"; do\n'
559 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
560 ' printf " %s" "$word"\n'
561 ' else\n'
562 ' printf " \'"\n'
563 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
564 ' printf "\'"\n'
565 ' fi\n'
566 'done >> "$this_log"\n'
567 'echo >> "$this_log"\n'
14f95a42
JM		 568 'echo "Directory: $dir" >> "$this_log"\n'
569 'echo "Path addition: $path" >> "$this_log"\n'
570 'echo >> "$this_log"\n'
571 'record_status ()\n'
572 '{\n'
573 ' echo >> "$this_log"\n'
574 ' echo "$1: $desc" > "$this_status"\n'
575 ' echo "$1: $desc" >> "$this_log"\n'
576 ' echo >> "$this_log"\n'
577 ' date >> "$this_log"\n'
578 ' echo "$1: $desc"\n'
579 ' exit 0\n'
580 '}\n'
581 'check_error ()\n'
582 '{\n'
583 ' if [ "$1" != "0" ]; then\n'
584 ' record_status FAIL\n'
585 ' fi\n'
586 '}\n'
587 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
588 ' record_status UNRESOLVED\n'
589 'fi\n'
590 'if [ "$dir" ]; then\n'
591 ' cd "$dir"\n'
592 ' check_error "$?"\n'
593 'fi\n'
594 'if [ "$path" ]; then\n'
595 ' PATH=$path:$PATH\n'
596 'fi\n'
597 '"$@" < /dev/null >> "$this_log" 2>&1\n'
598 'check_error "$?"\n'
599 'record_status PASS\n')
600 with open(self.wrapper, 'w') as f:
601 f.write(wrapper_text)
0c95f51d
JM		 602 # Mode 0o755.
603 mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
604 stat.S_IROTH|stat.S_IXOTH)
605 os.chmod(self.wrapper, mode_exec)
14f95a42
JM		 606 save_logs_text = (
607 '#!/bin/sh\n'
608 'if ! [ -f tests.sum ]; then\n'
609 ' echo "No test summary available."\n'
610 ' exit 0\n'
611 'fi\n'
612 'save_file ()\n'
613 '{\n'
614 ' echo "Contents of $1:"\n'
615 ' echo\n'
616 ' cat "$1"\n'
617 ' echo\n'
618 ' echo "End of contents of $1."\n'
619 ' echo\n'
620 '}\n'
621 'save_file tests.sum\n'
622 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
623 'for t in $non_pass_tests; do\n'
624 ' if [ -f "$t.out" ]; then\n'
625 ' save_file "$t.out"\n'
626 ' fi\n'
627 'done\n')
628 with open(self.save_logs, 'w') as f:
629 f.write(save_logs_text)
0c95f51d 630 os.chmod(self.save_logs, mode_exec)
14f95a42
JM		 631
632 def do_build(self):
633 """Do the actual build."""
634 cmd = ['make', '-j%d' % self.parallelism]
635 subprocess.run(cmd, cwd=self.builddir, check=True)
636
637 def build_host_libraries(self):
638 """Build the host libraries."""
639 installdir = self.host_libraries_installdir
640 builddir = os.path.join(self.builddir, 'host-libraries')
641 logsdir = os.path.join(self.logsdir, 'host-libraries')
642 self.remove_recreate_dirs(installdir, builddir, logsdir)
643 cmdlist = CommandList('host-libraries', self.keep)
644 self.build_host_library(cmdlist, 'gmp')
645 self.build_host_library(cmdlist, 'mpfr',
646 ['--with-gmp=%s' % installdir])
647 self.build_host_library(cmdlist, 'mpc',
648 ['--with-gmp=%s' % installdir,
649 '--with-mpfr=%s' % installdir])
650 cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
651 self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
652
653 def build_host_library(self, cmdlist, lib, extra_opts=None):
654 """Build one host library."""
655 srcdir = self.component_srcdir(lib)
656 builddir = self.component_builddir('host-libraries', None, lib)
657 installdir = self.host_libraries_installdir
658 cmdlist.push_subdesc(lib)
659 cmdlist.create_use_dir(builddir)
660 cfg_cmd = [os.path.join(srcdir, 'configure'),
661 '--prefix=%s' % installdir,
662 '--disable-shared']
663 if extra_opts:
664 cfg_cmd.extend (extra_opts)
665 cmdlist.add_command('configure', cfg_cmd)
666 cmdlist.add_command('build', ['make'])
667 cmdlist.add_command('check', ['make', 'check'])
668 cmdlist.add_command('install', ['make', 'install'])
669 cmdlist.cleanup_dir()
670 cmdlist.pop_subdesc()
671
672 def build_compilers(self, configs):
673 """Build the compilers."""
674 if not configs:
675 self.remove_dirs(os.path.join(self.builddir, 'compilers'))
676 self.remove_dirs(os.path.join(self.installdir, 'compilers'))
677 self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
678 configs = sorted(self.configs.keys())
679 for c in configs:
680 self.configs[c].build()
681
682 def build_glibcs(self, configs):
683 """Build the glibcs."""
684 if not configs:
685 self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
686 self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
687 self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
688 configs = sorted(self.glibc_configs.keys())
689 for c in configs:
690 self.glibc_configs[c].build()
691
02c78f02
JM		 692 def load_versions_json(self):
693 """Load information about source directory versions."""
694 if not os.access(self.versions_json, os.F_OK):
695 self.versions = {}
696 return
697 with open(self.versions_json, 'r') as f:
698 self.versions = json.load(f)
699
700 def store_json(self, data, filename):
701 """Store information in a JSON file."""
702 filename_tmp = filename + '.tmp'
703 with open(filename_tmp, 'w') as f:
704 json.dump(data, f, indent=2, sort_keys=True)
705 os.rename(filename_tmp, filename)
706
707 def store_versions_json(self):
708 """Store information about source directory versions."""
709 self.store_json(self.versions, self.versions_json)
710
711 def set_component_version(self, component, version, explicit, revision):
712 """Set the version information for a component."""
713 self.versions[component] = {'version': version,
714 'explicit': explicit,
715 'revision': revision}
716 self.store_versions_json()
717
14f95a42
JM		 718 def checkout(self, versions):
719 """Check out the desired component versions."""
b5b67ece 720 default_versions = {'binutils': 'vcs-2.33',
c4a392cf 721 'gcc': 'vcs-9',
14f95a42 722 'glibc': 'vcs-mainline',
f0107724 723 'gmp': '6.1.2',
17832eef 724 'linux': '5.4',
16d0f6ac 725 'mpc': '1.1.0',
932329a5 726 'mpfr': '4.0.2',
f4d3cee3 727 'mig': 'vcs-mainline',
19fd8d0d 728 'gnumach': 'vcs-mainline',
47c4b4b0 729 'hurd': 'vcs-mainline'}
14f95a42 730 use_versions = {}
02c78f02 731 explicit_versions = {}
14f95a42
JM		 732 for v in versions:
733 found_v = False
734 for k in default_versions.keys():
735 kx = k + '-'
736 if v.startswith(kx):
737 vx = v[len(kx):]
738 if k in use_versions:
739 print('error: multiple versions for %s' % k)
740 exit(1)
741 use_versions[k] = vx
02c78f02 742 explicit_versions[k] = True
14f95a42
JM		 743 found_v = True
744 break
745 if not found_v:
746 print('error: unknown component in %s' % v)
747 exit(1)
748 for k in default_versions.keys():
749 if k not in use_versions:
02c78f02
JM		 750 if k in self.versions and self.versions[k]['explicit']:
751 use_versions[k] = self.versions[k]['version']
752 explicit_versions[k] = True
753 else:
754 use_versions[k] = default_versions[k]
755 explicit_versions[k] = False
14f95a42
JM		 756 os.makedirs(self.srcdir, exist_ok=True)
757 for k in sorted(default_versions.keys()):
758 update = os.access(self.component_srcdir(k), os.F_OK)
759 v = use_versions[k]
02c78f02
JM		 760 if (update and
761 k in self.versions and
762 v != self.versions[k]['version']):
763 if not self.replace_sources:
764 print('error: version of %s has changed from %s to %s, '
765 'use --replace-sources to check out again' %
766 (k, self.versions[k]['version'], v))
767 exit(1)
768 shutil.rmtree(self.component_srcdir(k))
769 update = False
14f95a42 770 if v.startswith('vcs-'):
02c78f02 771 revision = self.checkout_vcs(k, v[4:], update)
14f95a42
JM		 772 else:
773 self.checkout_tar(k, v, update)
02c78f02
JM		 774 revision = v
775 self.set_component_version(k, v, explicit_versions[k], revision)
a1c9859b
JM		 776 if self.get_script_text() != self.script_text:
777 # Rerun the checkout process in case the updated script
778 # uses different default versions or new components.
779 self.exec_self()
14f95a42
JM		 780
781 def checkout_vcs(self, component, version, update):
782 """Check out the given version of the given component from version
02c78f02 783 control. Return a revision identifier."""
14f95a42
JM		 784 if component == 'binutils':
785 git_url = 'git://sourceware.org/git/binutils-gdb.git'
786 if version == 'mainline':
787 git_branch = 'master'
788 else:
789 trans = str.maketrans({'.': '_'})
790 git_branch = 'binutils-%s-branch' % version.translate(trans)
02c78f02 791 return self.git_checkout(component, git_url, git_branch, update)
14f95a42
JM		 792 elif component == 'gcc':
793 if version == 'mainline':
794 branch = 'trunk'
795 else:
796 trans = str.maketrans({'.': '_'})
797 branch = 'branches/gcc-%s-branch' % version.translate(trans)
798 svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
02c78f02 799 return self.gcc_checkout(svn_url, update)
14f95a42
JM		 800 elif component == 'glibc':
801 git_url = 'git://sourceware.org/git/glibc.git'
802 if version == 'mainline':
803 git_branch = 'master'
804 else:
805 git_branch = 'release/%s/master' % version
02c78f02 806 r = self.git_checkout(component, git_url, git_branch, update)
14f95a42 807 self.fix_glibc_timestamps()
02c78f02 808 return r
19fd8d0d
ST		 809 elif component == 'gnumach':
810 git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
811 git_branch = 'master'
812 r = self.git_checkout(component, git_url, git_branch, update)
813 subprocess.run(['autoreconf', '-i'],
814 cwd=self.component_srcdir(component), check=True)
815 return r
f4d3cee3
ST		 816 elif component == 'mig':
817 git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
818 git_branch = 'master'
819 r = self.git_checkout(component, git_url, git_branch, update)
820 subprocess.run(['autoreconf', '-i'],
821 cwd=self.component_srcdir(component), check=True)
822 return r
f9015003
ST		 823 elif component == 'hurd':
824 git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
825 git_branch = 'master'
48130690
ST		 826 r = self.git_checkout(component, git_url, git_branch, update)
827 subprocess.run(['autoconf'],
828 cwd=self.component_srcdir(component), check=True)
829 return r
14f95a42
JM		 830 else:
831 print('error: component %s coming from VCS' % component)
832 exit(1)
833
834 def git_checkout(self, component, git_url, git_branch, update):
02c78f02 835 """Check out a component from git. Return a commit identifier."""
14f95a42
JM		 836 if update:
837 subprocess.run(['git', 'remote', 'prune', 'origin'],
838 cwd=self.component_srcdir(component), check=True)
e3aa6999
JM		 839 if self.replace_sources:
840 subprocess.run(['git', 'clean', '-dxfq'],
841 cwd=self.component_srcdir(component), check=True)
14f95a42
JM		 842 subprocess.run(['git', 'pull', '-q'],
843 cwd=self.component_srcdir(component), check=True)
844 else:
845 subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
846 self.component_srcdir(component)], check=True)
02c78f02
JM		 847 r = subprocess.run(['git', 'rev-parse', 'HEAD'],
848 cwd=self.component_srcdir(component),
849 stdout=subprocess.PIPE,
850 check=True, universal_newlines=True).stdout
851 return r.rstrip()
14f95a42
JM		 852
853 def fix_glibc_timestamps(self):
854 """Fix timestamps in a glibc checkout."""
855 # Ensure that builds do not try to regenerate generated files
856 # in the source tree.
857 srcdir = self.component_srcdir('glibc')
2bd0bfcc
JM		 858 # These files have Makefile dependencies to regenerate them in
859 # the source tree that may be active during a normal build.
860 # Some other files have such dependencies but do not need to
861 # be touched because nothing in a build depends on the files
862 # in question.
053c52b1 863 for f in ('sysdeps/gnu/errlist.c',
5d9b7b9f 864 'sysdeps/mach/hurd/bits/errno.h'):
2bd0bfcc
JM		 865 to_touch = os.path.join(srcdir, f)
866 subprocess.run(['touch', '-c', to_touch], check=True)
14f95a42
JM		 867 for dirpath, dirnames, filenames in os.walk(srcdir):
868 for f in filenames:
869 if (f == 'configure' or
870 f == 'preconfigure' or
871 f.endswith('-kw.h')):
872 to_touch = os.path.join(dirpath, f)
873 subprocess.run(['touch', to_touch], check=True)
874
875 def gcc_checkout(self, svn_url, update):
02c78f02 876 """Check out GCC from SVN. Return the revision number."""
14f95a42
JM		 877 if not update:
878 subprocess.run(['svn', 'co', '-q', svn_url,
879 self.component_srcdir('gcc')], check=True)
880 subprocess.run(['contrib/gcc_update', '--silent'],
881 cwd=self.component_srcdir('gcc'), check=True)
02c78f02
JM		 882 r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
883 stdout=subprocess.PIPE,
884 check=True, universal_newlines=True).stdout
885 return r.rstrip()
14f95a42
JM		 886
887 def checkout_tar(self, component, version, update):
888 """Check out the given version of the given component from a
889 tarball."""
890 if update:
891 return
892 url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
c8fabb84 893 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
14f95a42 894 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
0c1041ee 895 'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
14f95a42 896 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
394b5bac
JM		 897 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
898 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
899 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
900 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
14f95a42
JM		 901 if component not in url_map:
902 print('error: component %s coming from tarball' % component)
903 exit(1)
0c1041ee
JM		 904 version_major = version.split('.')[0]
905 url = url_map[component] % {'version': version, 'major': version_major}
14f95a42
JM		 906 filename = os.path.join(self.srcdir, url.split('/')[-1])
907 response = urllib.request.urlopen(url)
908 data = response.read()
909 with open(filename, 'wb') as f:
910 f.write(data)
911 subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
912 check=True)
913 os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
914 self.component_srcdir(component))
915 os.remove(filename)
916
bf469f0c
JM		 917 def load_build_state_json(self):
918 """Load information about the state of previous builds."""
919 if os.access(self.build_state_json, os.F_OK):
920 with open(self.build_state_json, 'r') as f:
921 self.build_state = json.load(f)
922 else:
923 self.build_state = {}
924 for k in ('host-libraries', 'compilers', 'glibcs'):
925 if k not in self.build_state:
926 self.build_state[k] = {}
927 if 'build-time' not in self.build_state[k]:
928 self.build_state[k]['build-time'] = ''
929 if 'build-versions' not in self.build_state[k]:
930 self.build_state[k]['build-versions'] = {}
931 if 'build-results' not in self.build_state[k]:
932 self.build_state[k]['build-results'] = {}
933 if 'result-changes' not in self.build_state[k]:
934 self.build_state[k]['result-changes'] = {}
935 if 'ever-passed' not in self.build_state[k]:
936 self.build_state[k]['ever-passed'] = []
937
938 def store_build_state_json(self):
939 """Store information about the state of previous builds."""
940 self.store_json(self.build_state, self.build_state_json)
941
942 def clear_last_build_state(self, action):
943 """Clear information about the state of part of the build."""
944 # We clear the last build time and versions when starting a
945 # new build. The results of the last build are kept around,
946 # as comparison is still meaningful if this build is aborted
947 # and a new one started.
948 self.build_state[action]['build-time'] = ''
949 self.build_state[action]['build-versions'] = {}
950 self.store_build_state_json()
951
952 def update_build_state(self, action, build_time, build_versions):
953 """Update the build state after a build."""
954 build_time = build_time.replace(microsecond=0)
955 self.build_state[action]['build-time'] = str(build_time)
956 self.build_state[action]['build-versions'] = build_versions
957 build_results = {}
958 for log in self.status_log_list:
959 with open(log, 'r') as f:
960 log_text = f.read()
961 log_text = log_text.rstrip()
962 m = re.fullmatch('([A-Z]+): (.*)', log_text)
963 result = m.group(1)
964 test_name = m.group(2)
965 assert test_name not in build_results
966 build_results[test_name] = result
967 old_build_results = self.build_state[action]['build-results']
968 self.build_state[action]['build-results'] = build_results
969 result_changes = {}
970 all_tests = set(old_build_results.keys()) | set(build_results.keys())
971 for t in all_tests:
972 if t in old_build_results:
973 old_res = old_build_results[t]
974 else:
975 old_res = '(New test)'
976 if t in build_results:
977 new_res = build_results[t]
978 else:
979 new_res = '(Test removed)'
980 if old_res != new_res:
981 result_changes[t] = '%s -> %s' % (old_res, new_res)
982 self.build_state[action]['result-changes'] = result_changes
983 old_ever_passed = {t for t in self.build_state[action]['ever-passed']
984 if t in build_results}
985 new_passes = {t for t in build_results if build_results[t] == 'PASS'}
986 self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
987 new_passes)
988 self.store_build_state_json()
989
4d602bce
JM		 990 def load_bot_config_json(self):
991 """Load bot configuration."""
992 with open(self.bot_config_json, 'r') as f:
993 self.bot_config = json.load(f)
994
995 def part_build_old(self, action, delay):
996 """Return whether the last build for a given action was at least a
997 given number of seconds ago, or does not have a time recorded."""
998 old_time_str = self.build_state[action]['build-time']
999 if not old_time_str:
1000 return True
1001 old_time = datetime.datetime.strptime(old_time_str,
1002 '%Y-%m-%d %H:%M:%S')
1003 new_time = datetime.datetime.utcnow()
1004 delta = new_time - old_time
1005 return delta.total_seconds() >= delay
1006
1007 def bot_cycle(self):
1008 """Run a single round of checkout and builds."""
1009 print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1010 self.load_bot_config_json()
1011 actions = ('host-libraries', 'compilers', 'glibcs')
1012 self.bot_run_self(['--replace-sources'], 'checkout')
1013 self.load_versions_json()
1014 if self.get_script_text() != self.script_text:
1015 print('Script changed, re-execing.')
1016 # On script change, all parts of the build should be rerun.
1017 for a in actions:
1018 self.clear_last_build_state(a)
1019 self.exec_self()
1020 check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
394b5bac
JM		 1021 'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1022 'mig', 'gnumach', 'hurd'),
4d602bce
JM		 1023 'glibcs': ('glibc',)}
1024 must_build = {}
1025 for a in actions:
1026 build_vers = self.build_state[a]['build-versions']
1027 must_build[a] = False
1028 if not self.build_state[a]['build-time']:
1029 must_build[a] = True
1030 old_vers = {}
1031 new_vers = {}
1032 for c in check_components[a]:
1033 if c in build_vers:
1034 old_vers[c] = build_vers[c]
1035 new_vers[c] = {'version': self.versions[c]['version'],
1036 'revision': self.versions[c]['revision']}
1037 if new_vers == old_vers:
1038 print('Versions for %s unchanged.' % a)
1039 else:
1040 print('Versions changed or rebuild forced for %s.' % a)
1041 if a == 'compilers' and not self.part_build_old(
1042 a, self.bot_config['compilers-rebuild-delay']):
1043 print('Not requiring rebuild of compilers this soon.')
1044 else:
1045 must_build[a] = True
1046 if must_build['host-libraries']:
1047 must_build['compilers'] = True
1048 if must_build['compilers']:
1049 must_build['glibcs'] = True
1050 for a in actions:
1051 if must_build[a]:
1052 print('Must rebuild %s.' % a)
1053 self.clear_last_build_state(a)
1054 else:
1055 print('No need to rebuild %s.' % a)
a1f6a9ab
JM		 1056 if os.access(self.logsdir, os.F_OK):
1057 shutil.rmtree(self.logsdir_old, ignore_errors=True)
1058 shutil.copytree(self.logsdir, self.logsdir_old)
4d602bce
JM		 1059 for a in actions:
1060 if must_build[a]:
1061 build_time = datetime.datetime.utcnow()
1062 print('Rebuilding %s at %s.' % (a, str(build_time)))
1063 self.bot_run_self([], a)
1064 self.load_build_state_json()
1065 self.bot_build_mail(a, build_time)
1066 print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1067
1068 def bot_build_mail(self, action, build_time):
1069 """Send email with the results of a build."""
f0166c16
ZW		 1070 if not ('email-from' in self.bot_config and
1071 'email-server' in self.bot_config and
1072 'email-subject' in self.bot_config and
1073 'email-to' in self.bot_config):
1074 if not self.email_warning:
1075 print("Email not configured, not sending.")
1076 self.email_warning = True
1077 return
1078
4d602bce
JM		 1079 build_time = build_time.replace(microsecond=0)
1080 subject = (self.bot_config['email-subject'] %
1081 {'action': action,
1082 'build-time': str(build_time)})
1083 results = self.build_state[action]['build-results']
1084 changes = self.build_state[action]['result-changes']
1085 ever_passed = set(self.build_state[action]['ever-passed'])
1086 versions = self.build_state[action]['build-versions']
1087 new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1088 all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1089 all_fails = {k for k in results if results[k] == 'FAIL'}
1090 if new_regressions:
1091 new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1092 new_reg_text = ('New regressions:\n\n%s\n\n' %
1093 '\n'.join(new_reg_list))
1094 else:
1095 new_reg_text = ''
1096 if all_regressions:
1097 all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1098 all_reg_text = ('All regressions:\n\n%s\n\n' %
1099 '\n'.join(all_reg_list))
1100 else:
1101 all_reg_text = ''
1102 if all_fails:
1103 all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1104 all_fail_text = ('All failures:\n\n%s\n\n' %
1105 '\n'.join(all_fail_list))
1106 else:
1107 all_fail_text = ''
1108 if changes:
1109 changes_list = sorted(changes.keys())
1110 changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1111 changes_text = ('All changed results:\n\n%s\n\n' %
1112 '\n'.join(changes_list))
1113 else:
1114 changes_text = ''
1115 results_text = (new_reg_text + all_reg_text + all_fail_text +
1116 changes_text)
1117 if not results_text:
1118 results_text = 'Clean build with unchanged results.\n\n'
1119 versions_list = sorted(versions.keys())
1120 versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1121 versions[k]['revision'])
1122 for k in versions_list]
1123 versions_text = ('Component versions for this build:\n\n%s\n' %
1124 '\n'.join(versions_list))
1125 body_text = results_text + versions_text
1126 msg = email.mime.text.MIMEText(body_text)
1127 msg['Subject'] = subject
1128 msg['From'] = self.bot_config['email-from']
1129 msg['To'] = self.bot_config['email-to']
1130 msg['Message-ID'] = email.utils.make_msgid()
1131 msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1132 with smtplib.SMTP(self.bot_config['email-server']) as s:
1133 s.send_message(msg)
1134
a1f6a9ab 1135 def bot_run_self(self, opts, action, check=True):
4d602bce
JM		 1136 """Run a copy of this script with given options."""
1137 cmd = [sys.executable, sys.argv[0], '--keep=none',
1138 '-j%d' % self.parallelism]
b1176270
JM		 1139 if self.full_gcc:
1140 cmd.append('--full-gcc')
4d602bce
JM		 1141 cmd.extend(opts)
1142 cmd.extend([self.topdir, action])
a1f6a9ab
JM		 1143 sys.stdout.flush()
1144 subprocess.run(cmd, check=check)
1145
1146 def bot(self):
1147 """Run repeated rounds of checkout and builds."""
1148 while True:
1149 self.load_bot_config_json()
1150 if not self.bot_config['run']:
1151 print('Bot exiting by request.')
1152 exit(0)
1153 self.bot_run_self([], 'bot-cycle', check=False)
1154 self.load_bot_config_json()
1155 if not self.bot_config['run']:
1156 print('Bot exiting by request.')
1157 exit(0)
1158 time.sleep(self.bot_config['delay'])
1159 if self.get_script_text() != self.script_text:
1160 print('Script changed, bot re-execing.')
1161 self.exec_self()
4d602bce 1162
14f95a42 1163
0c95f51d 1164class Config(object):
14f95a42
JM		 1165 """A configuration for building a compiler and associated libraries."""
1166
1167 def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1168 first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1169 """Initialize a Config object."""
1170 self.ctx = ctx
1171 self.arch = arch
1172 self.os = os_name
1173 self.variant = variant
1174 if variant is None:
1175 self.name = '%s-%s' % (arch, os_name)
1176 else:
1177 self.name = '%s-%s-%s' % (arch, os_name, variant)
1178 self.triplet = '%s-glibc-%s' % (arch, os_name)
1179 if gcc_cfg is None:
1180 self.gcc_cfg = []
1181 else:
1182 self.gcc_cfg = gcc_cfg
1183 if first_gcc_cfg is None:
1184 self.first_gcc_cfg = []
1185 else:
1186 self.first_gcc_cfg = first_gcc_cfg
1187 if glibcs is None:
1188 glibcs = [{'variant': variant}]
1189 if extra_glibcs is None:
1190 extra_glibcs = []
1191 glibcs = [Glibc(self, **g) for g in glibcs]
1192 extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1193 self.all_glibcs = glibcs + extra_glibcs
1194 self.compiler_glibcs = glibcs
1195 self.installdir = ctx.compiler_installdir(self.name)
1196 self.bindir = ctx.compiler_bindir(self.name)
1197 self.sysroot = ctx.compiler_sysroot(self.name)
1198 self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1199 self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1200
1201 def component_builddir(self, component):
1202 """Return the directory to use for a (non-glibc) build."""
1203 return self.ctx.component_builddir('compilers', self.name, component)
1204
1205 def build(self):
1206 """Generate commands to build this compiler."""
1207 self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1208 self.logsdir)
1209 cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1210 cmdlist.add_command('check-host-libraries',
1211 ['test', '-f',
1212 os.path.join(self.ctx.host_libraries_installdir,
1213 'ok')])
1214 cmdlist.use_path(self.bindir)
1215 self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1216 ['--disable-gdb',
1217 '--disable-libdecnumber',
1218 '--disable-readline',
1219 '--disable-sim'])
1220 if self.os.startswith('linux'):
1221 self.install_linux_headers(cmdlist)
1222 self.build_gcc(cmdlist, True)
394b5bac
JM		 1223 if self.os == 'gnu':
1224 self.install_gnumach_headers(cmdlist)
1225 self.build_cross_tool(cmdlist, 'mig', 'mig')
1226 self.install_hurd_headers(cmdlist)
14f95a42
JM		 1227 for g in self.compiler_glibcs:
1228 cmdlist.push_subdesc('glibc')
1229 cmdlist.push_subdesc(g.name)
1230 g.build_glibc(cmdlist, True)
1231 cmdlist.pop_subdesc()
1232 cmdlist.pop_subdesc()
1233 self.build_gcc(cmdlist, False)
1234 cmdlist.add_command('done', ['touch',
1235 os.path.join(self.installdir, 'ok')])
1236 self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1237 self.logsdir)
1238
1239 def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1240 """Build one cross tool."""
1241 srcdir = self.ctx.component_srcdir(tool_src)
1242 builddir = self.component_builddir(tool_build)
1243 cmdlist.push_subdesc(tool_build)
1244 cmdlist.create_use_dir(builddir)
1245 cfg_cmd = [os.path.join(srcdir, 'configure'),
1246 '--prefix=%s' % self.installdir,
1247 '--build=%s' % self.ctx.build_triplet,
1248 '--host=%s' % self.ctx.build_triplet,
1249 '--target=%s' % self.triplet,
1250 '--with-sysroot=%s' % self.sysroot]
1251 if extra_opts:
1252 cfg_cmd.extend(extra_opts)
1253 cmdlist.add_command('configure', cfg_cmd)
1254 cmdlist.add_command('build', ['make'])
cd880aa2
JM		 1255 # Parallel "make install" for GCC has race conditions that can
1256 # cause it to fail; see
1257 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such
1258 # problems are not known for binutils, but doing the
1259 # installation in parallel within a particular toolchain build
1260 # (as opposed to installation of one toolchain from
1261 # build-many-glibcs.py running in parallel to the installation
1262 # of other toolchains being built) is not known to be
1263 # significantly beneficial, so it is simplest just to disable
1264 # parallel install for cross tools here.
1265 cmdlist.add_command('install', ['make', '-j1', 'install'])
14f95a42
JM		 1266 cmdlist.cleanup_dir()
1267 cmdlist.pop_subdesc()
1268
1269 def install_linux_headers(self, cmdlist):
1270 """Install Linux kernel headers."""
1271 arch_map = {'aarch64': 'arm64',
1272 'alpha': 'alpha',
1273 'arm': 'arm',
5f72b005 1274 'csky': 'csky',
14f95a42
JM		 1275 'hppa': 'parisc',
1276 'i486': 'x86',
1277 'i586': 'x86',
1278 'i686': 'x86',
1279 'i786': 'x86',
1280 'ia64': 'ia64',
1281 'm68k': 'm68k',
1282 'microblaze': 'microblaze',
1283 'mips': 'mips',
1284 'nios2': 'nios2',
1285 'powerpc': 'powerpc',
1286 's390': 's390',
0fd5d876
PD		 1287 'riscv32': 'riscv',
1288 'riscv64': 'riscv',
14f95a42
JM		 1289 'sh': 'sh',
1290 'sparc': 'sparc',
14f95a42
JM		 1291 'x86_64': 'x86'}
1292 linux_arch = None
1293 for k in arch_map:
1294 if self.arch.startswith(k):
1295 linux_arch = arch_map[k]
1296 break
1297 assert linux_arch is not None
1298 srcdir = self.ctx.component_srcdir('linux')
1299 builddir = self.component_builddir('linux')
1300 headers_dir = os.path.join(self.sysroot, 'usr')
1301 cmdlist.push_subdesc('linux')
1302 cmdlist.create_use_dir(builddir)
1303 cmdlist.add_command('install-headers',
1304 ['make', '-C', srcdir, 'O=%s' % builddir,
1305 'ARCH=%s' % linux_arch,
1306 'INSTALL_HDR_PATH=%s' % headers_dir,
1307 'headers_install'])
1308 cmdlist.cleanup_dir()
1309 cmdlist.pop_subdesc()
1310
394b5bac
JM		 1311 def install_gnumach_headers(self, cmdlist):
1312 """Install GNU Mach headers."""
1313 srcdir = self.ctx.component_srcdir('gnumach')
1314 builddir = self.component_builddir('gnumach')
1315 cmdlist.push_subdesc('gnumach')
1316 cmdlist.create_use_dir(builddir)
1317 cmdlist.add_command('configure',
1318 [os.path.join(srcdir, 'configure'),
1319 '--build=%s' % self.ctx.build_triplet,
1320 '--host=%s' % self.triplet,
1321 '--prefix=',
1322 'CC=%s-gcc -nostdlib' % self.triplet])
1323 cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1324 'install-data'])
1325 cmdlist.cleanup_dir()
1326 cmdlist.pop_subdesc()
1327
1328 def install_hurd_headers(self, cmdlist):
1329 """Install Hurd headers."""
1330 srcdir = self.ctx.component_srcdir('hurd')
1331 builddir = self.component_builddir('hurd')
1332 cmdlist.push_subdesc('hurd')
1333 cmdlist.create_use_dir(builddir)
1334 cmdlist.add_command('configure',
1335 [os.path.join(srcdir, 'configure'),
1336 '--build=%s' % self.ctx.build_triplet,
1337 '--host=%s' % self.triplet,
1338 '--prefix=',
1339 '--disable-profile', '--without-parted',
1340 'CC=%s-gcc -nostdlib' % self.triplet])
1341 cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1342 'no_deps=t', 'install-headers'])
1343 cmdlist.cleanup_dir()
1344 cmdlist.pop_subdesc()
1345
14f95a42
JM		 1346 def build_gcc(self, cmdlist, bootstrap):
1347 """Build GCC."""
b1176270
JM		 1348 # libssp is of little relevance with glibc's own stack
1349 # checking support. libcilkrts does not support GNU/Hurd (and
1350 # has been removed in GCC 8, so --disable-libcilkrts can be
1351 # removed once glibc no longer supports building with older
1352 # GCC versions).
14f95a42 1353 cfg_opts = list(self.gcc_cfg)
b1176270 1354 cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
c440d5d5
JM		 1355 host_libs = self.ctx.host_libraries_installdir
1356 cfg_opts += ['--with-gmp=%s' % host_libs,
1357 '--with-mpfr=%s' % host_libs,
1358 '--with-mpc=%s' % host_libs]
14f95a42
JM		 1359 if bootstrap:
1360 tool_build = 'gcc-first'
1361 # Building a static-only, C-only compiler that is
1362 # sufficient to build glibc. Various libraries and
1363 # features that may require libc headers must be disabled.
1364 # When configuring with a sysroot, --with-newlib is
1365 # required to define inhibit_libc (to stop some parts of
1366 # libgcc including libc headers); --without-headers is not
1367 # sufficient.
1368 cfg_opts += ['--enable-languages=c', '--disable-shared',
1369 '--disable-threads',
1370 '--disable-libatomic',
1371 '--disable-decimal-float',
1372 '--disable-libffi',
1373 '--disable-libgomp',
1374 '--disable-libitm',
1375 '--disable-libmpx',
1376 '--disable-libquadmath',
b1176270 1377 '--disable-libsanitizer',
14f95a42
JM		 1378 '--without-headers', '--with-newlib',
1379 '--with-glibc-version=%s' % self.ctx.glibc_version
1380 ]
1381 cfg_opts += self.first_gcc_cfg
1382 else:
1383 tool_build = 'gcc'
b1176270 1384 # libsanitizer commonly breaks because of glibc header
3aec0c39
FW		 1385 # changes, or on unusual targets. C++ pre-compiled
1386 # headers are not used during the glibc build and are
1387 # expensive to create.
b1176270 1388 if not self.ctx.full_gcc:
3aec0c39
FW		 1389 cfg_opts += ['--disable-libsanitizer',
1390 '--disable-libstdcxx-pch']
b1176270
JM		 1391 langs = 'all' if self.ctx.full_gcc else 'c,c++'
1392 cfg_opts += ['--enable-languages=%s' % langs,
1393 '--enable-shared', '--enable-threads']
14f95a42
JM		 1394 self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1395
1396
0c95f51d 1397class Glibc(object):
14f95a42
JM		 1398 """A configuration for building glibc."""
1399
1400 def __init__(self, compiler, arch=None, os_name=None, variant=None,
1401 cfg=None, ccopts=None):
1402 """Initialize a Glibc object."""
1403 self.ctx = compiler.ctx
1404 self.compiler = compiler
1405 if arch is None:
1406 self.arch = compiler.arch
1407 else:
1408 self.arch = arch
1409 if os_name is None:
1410 self.os = compiler.os
1411 else:
1412 self.os = os_name
1413 self.variant = variant
1414 if variant is None:
1415 self.name = '%s-%s' % (self.arch, self.os)
1416 else:
1417 self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1418 self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1419 if cfg is None:
1420 self.cfg = []
1421 else:
1422 self.cfg = cfg
1423 self.ccopts = ccopts
1424
1425 def tool_name(self, tool):
1426 """Return the name of a cross-compilation tool."""
1427 ctool = '%s-%s' % (self.compiler.triplet, tool)
1428 if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1429 ctool = '%s %s' % (ctool, self.ccopts)
1430 return ctool
1431
1432 def build(self):
1433 """Generate commands to build this glibc."""
1434 builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1435 installdir = self.ctx.glibc_installdir(self.name)
1436 logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1437 self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1438 cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1439 cmdlist.add_command('check-compilers',
1440 ['test', '-f',
1441 os.path.join(self.compiler.installdir, 'ok')])
1442 cmdlist.use_path(self.compiler.bindir)
1443 self.build_glibc(cmdlist, False)
1444 self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1445 logsdir)
1446
1447 def build_glibc(self, cmdlist, for_compiler):
1448 """Generate commands to build this glibc, either as part of a compiler
1449 build or with the bootstrapped compiler (and in the latter case, run
1450 tests as well)."""
1451 srcdir = self.ctx.component_srcdir('glibc')
1452 if for_compiler:
1453 builddir = self.ctx.component_builddir('compilers',
1454 self.compiler.name, 'glibc',
1455 self.name)
1456 installdir = self.compiler.sysroot
14f95a42
JM		 1457 else:
1458 builddir = self.ctx.component_builddir('glibcs', self.name,
1459 'glibc')
1460 installdir = self.ctx.glibc_installdir(self.name)
14f95a42 1461 cmdlist.create_use_dir(builddir)
394b5bac
JM		 1462 use_usr = self.os != 'gnu'
1463 prefix = '/usr' if use_usr else ''
530504e3 1464 cfg_cmd = [os.path.join(srcdir, 'configure'),
394b5bac 1465 '--prefix=%s' % prefix,
218bb835 1466 '--enable-profile',
14f95a42
JM		 1467 '--build=%s' % self.ctx.build_triplet,
1468 '--host=%s' % self.triplet,
1469 'CC=%s' % self.tool_name('gcc'),
1470 'CXX=%s' % self.tool_name('g++'),
1471 'AR=%s' % self.tool_name('ar'),
1472 'AS=%s' % self.tool_name('as'),
1473 'LD=%s' % self.tool_name('ld'),
1474 'NM=%s' % self.tool_name('nm'),
1475 'OBJCOPY=%s' % self.tool_name('objcopy'),
1476 'OBJDUMP=%s' % self.tool_name('objdump'),
1477 'RANLIB=%s' % self.tool_name('ranlib'),
1478 'READELF=%s' % self.tool_name('readelf'),
1479 'STRIP=%s' % self.tool_name('strip')]
394b5bac
JM		 1480 if self.os == 'gnu':
1481 cfg_cmd += ['MIG=%s' % self.tool_name('mig')]
14f95a42
JM		 1482 cfg_cmd += self.cfg
1483 cmdlist.add_command('configure', cfg_cmd)
1484 cmdlist.add_command('build', ['make'])
1485 cmdlist.add_command('install', ['make', 'install',
1486 'install_root=%s' % installdir])
1487 # GCC uses paths such as lib/../lib64, so make sure lib
1488 # directories always exist.
394b5bac
JM		 1489 mkdir_cmd = ['mkdir', '-p',
1490 os.path.join(installdir, 'lib')]
1491 if use_usr:
1492 mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')]
1493 cmdlist.add_command('mkdir-lib', mkdir_cmd)
14f95a42 1494 if not for_compiler:
297635d8
JM		 1495 if self.ctx.strip:
1496 cmdlist.add_command('strip',
1497 ['sh', '-c',
1598f3ad 1498 ('%s $(find %s/lib* -name "*.so")' %
297635d8 1499 (self.tool_name('strip'), installdir))])
14f95a42
JM		 1500 cmdlist.add_command('check', ['make', 'check'])
1501 cmdlist.add_command('save-logs', [self.ctx.save_logs],
1502 always_run=True)
14f95a42
JM		 1503 cmdlist.cleanup_dir()
1504
1505
0c95f51d 1506class Command(object):
14f95a42
JM		 1507 """A command run in the build process."""
1508
1509 def __init__(self, desc, num, dir, path, command, always_run=False):
1510 """Initialize a Command object."""
1511 self.dir = dir
1512 self.path = path
1513 self.desc = desc
1514 trans = str.maketrans({' ': '-'})
1515 self.logbase = '%03d-%s' % (num, desc.translate(trans))
1516 self.command = command
1517 self.always_run = always_run
1518
1519 @staticmethod
1520 def shell_make_quote_string(s):
1521 """Given a string not containing a newline, quote it for use by the
1522 shell and make."""
1523 assert '\n' not in s
1524 if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1525 return s
1526 strans = str.maketrans({"'": "'\\''"})
1527 s = "'%s'" % s.translate(strans)
1528 mtrans = str.maketrans({'$': '$$'})
1529 return s.translate(mtrans)
1530
1531 @staticmethod
1532 def shell_make_quote_list(l, translate_make):
1533 """Given a list of strings not containing newlines, quote them for use
1534 by the shell and make, returning a single string. If translate_make
1535 is true and the first string is 'make', change it to $(MAKE)."""
1536 l = [Command.shell_make_quote_string(s) for s in l]
1537 if translate_make and l[0] == 'make':
1538 l[0] = '$(MAKE)'
1539 return ' '.join(l)
1540
1541 def shell_make_quote(self):
1542 """Return this command quoted for the shell and make."""
1543 return self.shell_make_quote_list(self.command, True)
1544
1545
0c95f51d 1546class CommandList(object):
14f95a42
JM		 1547 """A list of commands run in the build process."""
1548
1549 def __init__(self, desc, keep):
1550 """Initialize a CommandList object."""
1551 self.cmdlist = []
1552 self.dir = None
1553 self.path = None
1554 self.desc = [desc]
1555 self.keep = keep
1556
1557 def desc_txt(self, desc):
1558 """Return the description to use for a command."""
1559 return '%s %s' % (' '.join(self.desc), desc)
1560
1561 def use_dir(self, dir):
1562 """Set the default directory for subsequent commands."""
1563 self.dir = dir
1564
1565 def use_path(self, path):
1566 """Set a directory to be prepended to the PATH for subsequent
1567 commands."""
1568 self.path = path
1569
1570 def push_subdesc(self, subdesc):
1571 """Set the default subdescription for subsequent commands (e.g., the
1572 name of a component being built, within the series of commands
1573 building it)."""
1574 self.desc.append(subdesc)
1575
1576 def pop_subdesc(self):
1577 """Pop a subdescription from the list of descriptions."""
1578 self.desc.pop()
1579
1580 def create_use_dir(self, dir):
1581 """Remove and recreate a directory and use it for subsequent
1582 commands."""
1583 self.add_command_dir('rm', None, ['rm', '-rf', dir])
1584 self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1585 self.use_dir(dir)
1586
14f95a42
JM		 1587 def add_command_dir(self, desc, dir, command, always_run=False):
1588 """Add a command to run in a given directory."""
1589 cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1590 command, always_run)
1591 self.cmdlist.append(cmd)
1592
1593 def add_command(self, desc, command, always_run=False):
1594 """Add a command to run in the default directory."""
1595 cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1596 self.path, command, always_run)
1597 self.cmdlist.append(cmd)
1598
1599 def cleanup_dir(self, desc='cleanup', dir=None):
1600 """Clean up a build directory. If no directory is specified, the
1601 default directory is cleaned up and ceases to be the default
1602 directory."""
1603 if dir is None:
1604 dir = self.dir
1605 self.use_dir(None)
1606 if self.keep != 'all':
1607 self.add_command_dir(desc, None, ['rm', '-rf', dir],
1608 always_run=(self.keep == 'none'))
1609
1610 def makefile_commands(self, wrapper, logsdir):
1611 """Return the sequence of commands in the form of text for a Makefile.
1612 The given wrapper script takes arguments: base of logs for
1613 previous command, or empty; base of logs for this command;
1614 description; directory; PATH addition; the command itself."""
1615 # prev_base is the base of the name for logs of the previous
1616 # command that is not always-run (that is, a build command,
1617 # whose failure should stop subsequent build commands from
1618 # being run, as opposed to a cleanup command, which is run
1619 # even if previous commands failed).
1620 prev_base = ''
1621 cmds = []
1622 for c in self.cmdlist:
1623 ctxt = c.shell_make_quote()
1624 if prev_base and not c.always_run:
1625 prev_log = os.path.join(logsdir, prev_base)
1626 else:
1627 prev_log = ''
1628 this_log = os.path.join(logsdir, c.logbase)
1629 if not c.always_run:
1630 prev_base = c.logbase
1631 if c.dir is None:
1632 dir = ''
1633 else:
1634 dir = c.dir
1635 if c.path is None:
1636 path = ''
1637 else:
1638 path = c.path
1639 prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1640 prelim_txt = Command.shell_make_quote_list(prelims, False)
1641 cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1642 return '\n'.join(cmds)
1643
bf469f0c
JM		 1644 def status_logs(self, logsdir):
1645 """Return the list of log files with command status."""
1646 return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1647 for c in self.cmdlist]
1648
14f95a42
JM		 1649
1650def get_parser():
1651 """Return an argument parser for this module."""
1652 parser = argparse.ArgumentParser(description=__doc__)
1653 parser.add_argument('-j', dest='parallelism',
1654 help='Run this number of jobs in parallel',
1655 type=int, default=os.cpu_count())
1656 parser.add_argument('--keep', dest='keep',
1657 help='Whether to keep all build directories, '
1658 'none or only those from failed builds',
1659 default='none', choices=('none', 'all', 'failed'))
02c78f02
JM		 1660 parser.add_argument('--replace-sources', action='store_true',
1661 help='Remove and replace source directories '
1662 'with the wrong version of a component')
297635d8
JM		 1663 parser.add_argument('--strip', action='store_true',
1664 help='Strip installed glibc libraries')
b1176270
JM		 1665 parser.add_argument('--full-gcc', action='store_true',
1666 help='Build GCC with all languages and libsanitizer')
14f95a42
JM		 1667 parser.add_argument('topdir',
1668 help='Toplevel working directory')
1669 parser.add_argument('action',
1670 help='What to do',
a1f6a9ab
JM		 1671 choices=('checkout', 'bot-cycle', 'bot',
1672 'host-libraries', 'compilers', 'glibcs'))
14f95a42
JM		 1673 parser.add_argument('configs',
1674 help='Versions to check out or configurations to build',
1675 nargs='*')
1676 return parser
1677
1678
1679def main(argv):
1680 """The main entry point."""
1681 parser = get_parser()
1682 opts = parser.parse_args(argv)
1683 topdir = os.path.abspath(opts.topdir)
02c78f02 1684 ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
b1176270 1685 opts.strip, opts.full_gcc, opts.action)
14f95a42
JM		 1686 ctx.run_builds(opts.action, opts.configs)
1687
1688
1689if __name__ == '__main__':
1690 main(sys.argv[1:])
A mirror of the official glibc repository