]> git.ipfire.org Git - thirdparty/glibc.git/blame - scripts/build-many-glibcs.py
Use Linux 6.0 in build-many-glibcs.py
[thirdparty/glibc.git] / scripts / build-many-glibcs.py
CommitLineData
14f95a42
JM
1#!/usr/bin/python3
2# Build many configurations of glibc.
581c785b 3# Copyright (C) 2016-2022 Free Software Foundation, Inc.
72e4a717 4# Copyright The GNU Toolchain Authors.
14f95a42
JM
5# This file is part of the GNU C Library.
6#
7# The GNU C Library is free software; you can redistribute it and/or
8# modify it under the terms of the GNU Lesser General Public
9# License as published by the Free Software Foundation; either
10# version 2.1 of the License, or (at your option) any later version.
11#
12# The GNU C Library is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15# Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public
18# License along with the GNU C Library; if not, see
5a82c748 19# <https://www.gnu.org/licenses/>.
14f95a42
JM
20
21"""Build many configurations of glibc.
22
23This script takes as arguments a directory name (containing a src
24subdirectory with sources of the relevant toolchain components) and a
25description of what to do: 'checkout', to check out sources into that
4d602bce 26directory, 'bot-cycle', to run a series of checkout and build steps,
a1f6a9ab
JM
27'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
28libraries required by the toolchain, 'compilers', to build
29cross-compilers for various configurations, or 'glibcs', to build
30glibc for various configurations and run the compilation parts of the
31testsuite. Subsequent arguments name the versions of components to
32check out (<component>-<version), for 'checkout', or, for actions
33other than 'checkout' and 'bot-cycle', name configurations for which
34compilers or glibc are to be built.
4d602bce 35
feaa1506
FW
36The 'list-compilers' command prints the name of each available
37compiler configuration, without building anything. The 'list-glibcs'
38command prints the name of each glibc compiler configuration, followed
39by the space, followed by the name of the compiler configuration used
40for building this glibc variant.
41
14f95a42
JM
42"""
43
44import argparse
bf469f0c 45import datetime
4d602bce
JM
46import email.mime.text
47import email.utils
02c78f02 48import json
14f95a42 49import os
14f95a42
JM
50import re
51import shutil
4d602bce 52import smtplib
14f95a42
JM
53import stat
54import subprocess
55import sys
a1f6a9ab 56import time
14f95a42
JM
57import urllib.request
58
64235ccc
SN
59try:
60 subprocess.run
61except:
62 class _CompletedProcess:
63 def __init__(self, args, returncode, stdout=None, stderr=None):
64 self.args = args
65 self.returncode = returncode
66 self.stdout = stdout
67 self.stderr = stderr
68
69 def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
70 assert(timeout is None)
71 with subprocess.Popen(*popenargs, **kwargs) as process:
72 try:
73 stdout, stderr = process.communicate(input)
74 except:
75 process.kill()
76 process.wait()
77 raise
78 returncode = process.poll()
79 if check and returncode:
80 raise subprocess.CalledProcessError(returncode, popenargs)
81 return _CompletedProcess(popenargs, returncode, stdout, stderr)
82
83 subprocess.run = _run
84
14f95a42 85
0c95f51d 86class Context(object):
14f95a42
JM
87 """The global state associated with builds in a given directory."""
88
297635d8 89 def __init__(self, topdir, parallelism, keep, replace_sources, strip,
c592721a 90 full_gcc, action, shallow=False):
14f95a42
JM
91 """Initialize the context."""
92 self.topdir = topdir
93 self.parallelism = parallelism
94 self.keep = keep
02c78f02 95 self.replace_sources = replace_sources
297635d8 96 self.strip = strip
b1176270 97 self.full_gcc = full_gcc
c592721a 98 self.shallow = shallow
14f95a42 99 self.srcdir = os.path.join(topdir, 'src')
02c78f02 100 self.versions_json = os.path.join(self.srcdir, 'versions.json')
bf469f0c 101 self.build_state_json = os.path.join(topdir, 'build-state.json')
4d602bce 102 self.bot_config_json = os.path.join(topdir, 'bot-config.json')
14f95a42
JM
103 self.installdir = os.path.join(topdir, 'install')
104 self.host_libraries_installdir = os.path.join(self.installdir,
105 'host-libraries')
106 self.builddir = os.path.join(topdir, 'build')
107 self.logsdir = os.path.join(topdir, 'logs')
a1f6a9ab 108 self.logsdir_old = os.path.join(topdir, 'logs-old')
14f95a42
JM
109 self.makefile = os.path.join(self.builddir, 'Makefile')
110 self.wrapper = os.path.join(self.builddir, 'wrapper')
111 self.save_logs = os.path.join(self.builddir, 'save-logs')
a1c9859b 112 self.script_text = self.get_script_text()
feaa1506 113 if action not in ('checkout', 'list-compilers', 'list-glibcs'):
14f95a42
JM
114 self.build_triplet = self.get_build_triplet()
115 self.glibc_version = self.get_glibc_version()
116 self.configs = {}
117 self.glibc_configs = {}
118 self.makefile_pieces = ['.PHONY: all\n']
119 self.add_all_configs()
02c78f02 120 self.load_versions_json()
bf469f0c
JM
121 self.load_build_state_json()
122 self.status_log_list = []
f0166c16 123 self.email_warning = False
14f95a42 124
a1c9859b
JM
125 def get_script_text(self):
126 """Return the text of this script."""
127 with open(sys.argv[0], 'r') as f:
128 return f.read()
129
130 def exec_self(self):
131 """Re-execute this script with the same arguments."""
36820ce9 132 sys.stdout.flush()
a1c9859b
JM
133 os.execv(sys.executable, [sys.executable] + sys.argv)
134
14f95a42
JM
135 def get_build_triplet(self):
136 """Determine the build triplet with config.guess."""
137 config_guess = os.path.join(self.component_srcdir('gcc'),
138 'config.guess')
139 cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
140 check=True, universal_newlines=True).stdout
141 return cg_out.rstrip()
142
143 def get_glibc_version(self):
144 """Determine the glibc version number (major.minor)."""
145 version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
146 with open(version_h, 'r') as f:
147 lines = f.readlines()
148 starttext = '#define VERSION "'
149 for l in lines:
150 if l.startswith(starttext):
151 l = l[len(starttext):]
152 l = l.rstrip('"\n')
153 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
154 return '%s.%s' % m.group(1, 2)
155 print('error: could not determine glibc version')
156 exit(1)
157
158 def add_all_configs(self):
159 """Add all known glibc build configurations."""
160 self.add_config(arch='aarch64',
6cb86fd2
SN
161 os_name='linux-gnu',
162 extra_glibcs=[{'variant': 'disable-multi-arch',
163 'cfg': ['--disable-multi-arch']}])
14f95a42
JM
164 self.add_config(arch='aarch64_be',
165 os_name='linux-gnu')
2fc2260b
VG
166 self.add_config(arch='arc',
167 os_name='linux-gnu',
168 gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
169 self.add_config(arch='arc',
170 os_name='linux-gnuhf',
171 gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
172 self.add_config(arch='arceb',
173 os_name='linux-gnu',
174 gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
14f95a42
JM
175 self.add_config(arch='alpha',
176 os_name='linux-gnu')
177 self.add_config(arch='arm',
72edea80
FW
178 os_name='linux-gnueabi',
179 extra_glibcs=[{'variant': 'v4t',
180 'ccopts': '-march=armv4t'}])
14f95a42
JM
181 self.add_config(arch='armeb',
182 os_name='linux-gnueabi')
183 self.add_config(arch='armeb',
184 os_name='linux-gnueabi',
185 variant='be8',
186 gcc_cfg=['--with-arch=armv7-a'])
187 self.add_config(arch='arm',
3d265601 188 os_name='linux-gnueabihf',
8e52f573 189 gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
3d265601 190 extra_glibcs=[{'variant': 'v7a',
8e52f573 191 'ccopts': '-march=armv7-a -mfpu=vfpv3'},
1e5235c5
FW
192 {'variant': 'thumb',
193 'ccopts':
194 '-mthumb -march=armv7-a -mfpu=vfpv3'},
3d265601 195 {'variant': 'v7a-disable-multi-arch',
8e52f573 196 'ccopts': '-march=armv7-a -mfpu=vfpv3',
3d265601 197 'cfg': ['--disable-multi-arch']}])
14f95a42 198 self.add_config(arch='armeb',
5170fa49 199 os_name='linux-gnueabihf',
8e52f573 200 gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
14f95a42
JM
201 self.add_config(arch='armeb',
202 os_name='linux-gnueabihf',
203 variant='be8',
8e52f573
JM
204 gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
205 '--with-fpu=vfpv3'])
5f72b005
MH
206 self.add_config(arch='csky',
207 os_name='linux-gnuabiv2',
208 variant='soft',
209 gcc_cfg=['--disable-multilib'])
210 self.add_config(arch='csky',
211 os_name='linux-gnuabiv2',
212 gcc_cfg=['--with-float=hard', '--disable-multilib'])
14f95a42
JM
213 self.add_config(arch='hppa',
214 os_name='linux-gnu')
394b5bac
JM
215 self.add_config(arch='i686',
216 os_name='gnu')
14f95a42
JM
217 self.add_config(arch='ia64',
218 os_name='linux-gnu',
23a7896d
JM
219 first_gcc_cfg=['--with-system-libunwind'],
220 binutils_cfg=['--enable-obsolete'])
3be5fc00 221 self.add_config(arch='loongarch64',
222 os_name='linux-gnu',
223 variant='lp64d',
224 gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
93099392 225 self.add_config(arch='loongarch64',
226 os_name='linux-gnu',
227 variant='lp64s',
228 gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
14f95a42
JM
229 self.add_config(arch='m68k',
230 os_name='linux-gnu',
231 gcc_cfg=['--disable-multilib'])
232 self.add_config(arch='m68k',
233 os_name='linux-gnu',
234 variant='coldfire',
235 gcc_cfg=['--with-arch=cf', '--disable-multilib'])
3005b705
JM
236 self.add_config(arch='m68k',
237 os_name='linux-gnu',
238 variant='coldfire-soft',
239 gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
240 '--disable-multilib'])
14f95a42
JM
241 self.add_config(arch='microblaze',
242 os_name='linux-gnu',
243 gcc_cfg=['--disable-multilib'])
244 self.add_config(arch='microblazeel',
245 os_name='linux-gnu',
246 gcc_cfg=['--disable-multilib'])
247 self.add_config(arch='mips64',
248 os_name='linux-gnu',
249 gcc_cfg=['--with-mips-plt'],
250 glibcs=[{'variant': 'n32'},
251 {'arch': 'mips',
252 'ccopts': '-mabi=32'},
253 {'variant': 'n64',
254 'ccopts': '-mabi=64'}])
255 self.add_config(arch='mips64',
256 os_name='linux-gnu',
257 variant='soft',
258 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
8df5d347 259 glibcs=[{'variant': 'n32-soft'},
14f95a42
JM
260 {'variant': 'soft',
261 'arch': 'mips',
8df5d347 262 'ccopts': '-mabi=32'},
14f95a42 263 {'variant': 'n64-soft',
8df5d347 264 'ccopts': '-mabi=64'}])
14f95a42
JM
265 self.add_config(arch='mips64',
266 os_name='linux-gnu',
267 variant='nan2008',
268 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
269 '--with-arch-64=mips64r2',
270 '--with-arch-32=mips32r2'],
271 glibcs=[{'variant': 'n32-nan2008'},
272 {'variant': 'nan2008',
273 'arch': 'mips',
274 'ccopts': '-mabi=32'},
275 {'variant': 'n64-nan2008',
276 'ccopts': '-mabi=64'}])
277 self.add_config(arch='mips64',
278 os_name='linux-gnu',
279 variant='nan2008-soft',
280 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
281 '--with-arch-64=mips64r2',
282 '--with-arch-32=mips32r2',
283 '--with-float=soft'],
8df5d347 284 glibcs=[{'variant': 'n32-nan2008-soft'},
14f95a42
JM
285 {'variant': 'nan2008-soft',
286 'arch': 'mips',
8df5d347 287 'ccopts': '-mabi=32'},
14f95a42 288 {'variant': 'n64-nan2008-soft',
8df5d347 289 'ccopts': '-mabi=64'}])
14f95a42
JM
290 self.add_config(arch='mips64el',
291 os_name='linux-gnu',
292 gcc_cfg=['--with-mips-plt'],
293 glibcs=[{'variant': 'n32'},
294 {'arch': 'mipsel',
295 'ccopts': '-mabi=32'},
296 {'variant': 'n64',
297 'ccopts': '-mabi=64'}])
298 self.add_config(arch='mips64el',
299 os_name='linux-gnu',
300 variant='soft',
301 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
8df5d347 302 glibcs=[{'variant': 'n32-soft'},
14f95a42
JM
303 {'variant': 'soft',
304 'arch': 'mipsel',
8df5d347 305 'ccopts': '-mabi=32'},
14f95a42 306 {'variant': 'n64-soft',
8df5d347 307 'ccopts': '-mabi=64'}])
14f95a42
JM
308 self.add_config(arch='mips64el',
309 os_name='linux-gnu',
310 variant='nan2008',
311 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
312 '--with-arch-64=mips64r2',
313 '--with-arch-32=mips32r2'],
314 glibcs=[{'variant': 'n32-nan2008'},
315 {'variant': 'nan2008',
316 'arch': 'mipsel',
317 'ccopts': '-mabi=32'},
318 {'variant': 'n64-nan2008',
319 'ccopts': '-mabi=64'}])
320 self.add_config(arch='mips64el',
321 os_name='linux-gnu',
322 variant='nan2008-soft',
323 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
324 '--with-arch-64=mips64r2',
325 '--with-arch-32=mips32r2',
326 '--with-float=soft'],
8df5d347 327 glibcs=[{'variant': 'n32-nan2008-soft'},
14f95a42
JM
328 {'variant': 'nan2008-soft',
329 'arch': 'mipsel',
8df5d347 330 'ccopts': '-mabi=32'},
14f95a42 331 {'variant': 'n64-nan2008-soft',
8df5d347 332 'ccopts': '-mabi=64'}])
a2e487ce
DM
333 self.add_config(arch='mipsisa64r6el',
334 os_name='linux-gnu',
335 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
336 '--with-arch-64=mips64r6',
337 '--with-arch-32=mips32r6',
338 '--with-float=hard'],
339 glibcs=[{'variant': 'n32'},
340 {'arch': 'mipsisa32r6el',
341 'ccopts': '-mabi=32'},
342 {'variant': 'n64',
343 'ccopts': '-mabi=64'}])
14f95a42
JM
344 self.add_config(arch='nios2',
345 os_name='linux-gnu')
e9816506
SH
346 self.add_config(arch='or1k',
347 os_name='linux-gnu',
348 variant='soft',
349 gcc_cfg=['--with-multilib-list=mcmov'])
14f95a42
JM
350 self.add_config(arch='powerpc',
351 os_name='linux-gnu',
4179178b
JM
352 gcc_cfg=['--disable-multilib', '--enable-secureplt'],
353 extra_glibcs=[{'variant': 'power4',
354 'ccopts': '-mcpu=power4',
355 'cfg': ['--with-cpu=power4']}])
14f95a42
JM
356 self.add_config(arch='powerpc',
357 os_name='linux-gnu',
358 variant='soft',
359 gcc_cfg=['--disable-multilib', '--with-float=soft',
8df5d347 360 '--enable-secureplt'])
14f95a42
JM
361 self.add_config(arch='powerpc64',
362 os_name='linux-gnu',
363 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
364 self.add_config(arch='powerpc64le',
365 os_name='linux-gnu',
4ae2b8a4
PM
366 gcc_cfg=['--disable-multilib', '--enable-secureplt'],
367 extra_glibcs=[{'variant': 'disable-multi-arch',
368 'cfg': ['--disable-multi-arch']}])
567b1705
ZL
369 self.add_config(arch='riscv32',
370 os_name='linux-gnu',
371 variant='rv32imac-ilp32',
372 gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
373 '--disable-multilib'])
374 self.add_config(arch='riscv32',
375 os_name='linux-gnu',
376 variant='rv32imafdc-ilp32',
377 gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
378 '--disable-multilib'])
379 self.add_config(arch='riscv32',
380 os_name='linux-gnu',
381 variant='rv32imafdc-ilp32d',
382 gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
383 '--disable-multilib'])
0fd5d876
PD
384 self.add_config(arch='riscv64',
385 os_name='linux-gnu',
386 variant='rv64imac-lp64',
387 gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
388 '--disable-multilib'])
389 self.add_config(arch='riscv64',
390 os_name='linux-gnu',
391 variant='rv64imafdc-lp64',
392 gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
393 '--disable-multilib'])
394 self.add_config(arch='riscv64',
395 os_name='linux-gnu',
396 variant='rv64imafdc-lp64d',
397 gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
398 '--disable-multilib'])
14f95a42
JM
399 self.add_config(arch='s390x',
400 os_name='linux-gnu',
401 glibcs=[{},
33dd32fd
SL
402 {'arch': 's390', 'ccopts': '-m31'}],
403 extra_glibcs=[{'variant': 'O3',
404 'cflags': '-O3'}])
14f95a42 405 self.add_config(arch='sh3',
c89721e2 406 os_name='linux-gnu')
14f95a42 407 self.add_config(arch='sh3eb',
c89721e2 408 os_name='linux-gnu')
14f95a42 409 self.add_config(arch='sh4',
c89721e2 410 os_name='linux-gnu')
14f95a42 411 self.add_config(arch='sh4eb',
c89721e2 412 os_name='linux-gnu')
14f95a42
JM
413 self.add_config(arch='sh4',
414 os_name='linux-gnu',
415 variant='soft',
8df5d347 416 gcc_cfg=['--without-fp'])
14f95a42
JM
417 self.add_config(arch='sh4eb',
418 os_name='linux-gnu',
419 variant='soft',
8df5d347 420 gcc_cfg=['--without-fp'])
14f95a42
JM
421 self.add_config(arch='sparc64',
422 os_name='linux-gnu',
423 glibcs=[{},
424 {'arch': 'sparcv9',
5d9b7b9f 425 'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
066d80a7
AZ
426 extra_glibcs=[{'variant': 'leon3',
427 'arch' : 'sparcv8',
428 'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
429 {'variant': 'disable-multi-arch',
0c097378
JM
430 'cfg': ['--disable-multi-arch']},
431 {'variant': 'disable-multi-arch',
432 'arch': 'sparcv9',
066d80a7 433 'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
0c097378 434 'cfg': ['--disable-multi-arch']}])
14f95a42
JM
435 self.add_config(arch='x86_64',
436 os_name='linux-gnu',
437 gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
438 glibcs=[{},
439 {'variant': 'x32', 'ccopts': '-mx32'},
440 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
441 extra_glibcs=[{'variant': 'disable-multi-arch',
442 'cfg': ['--disable-multi-arch']},
5f18453d
FW
443 {'variant': 'minimal',
444 'cfg': ['--disable-multi-arch',
445 '--disable-profile',
446 '--disable-timezone-tools',
447 '--disable-mathvec',
448 '--disable-tunables',
449 '--disable-crypt',
450 '--disable-experimental-malloc',
451 '--disable-build-nscd',
452 '--disable-nscd']},
23645707
SP
453 {'variant': 'no-pie',
454 'cfg': ['--disable-default-pie']},
455 {'variant': 'x32-no-pie',
1a49fc59 456 'ccopts': '-mx32',
23645707
SP
457 'cfg': ['--disable-default-pie']},
458 {'variant': 'no-pie',
1a49fc59
L
459 'arch': 'i686',
460 'ccopts': '-m32 -march=i686',
23645707 461 'cfg': ['--disable-default-pie']},
14f95a42
JM
462 {'variant': 'disable-multi-arch',
463 'arch': 'i686',
464 'ccopts': '-m32 -march=i686',
465 'cfg': ['--disable-multi-arch']},
466 {'arch': 'i486',
467 'ccopts': '-m32 -march=i486'},
468 {'arch': 'i586',
469 'ccopts': '-m32 -march=i586'}])
470
471 def add_config(self, **args):
472 """Add an individual build configuration."""
473 cfg = Config(self, **args)
474 if cfg.name in self.configs:
475 print('error: duplicate config %s' % cfg.name)
476 exit(1)
477 self.configs[cfg.name] = cfg
478 for c in cfg.all_glibcs:
479 if c.name in self.glibc_configs:
480 print('error: duplicate glibc config %s' % c.name)
481 exit(1)
482 self.glibc_configs[c.name] = c
483
484 def component_srcdir(self, component):
485 """Return the source directory for a given component, e.g. gcc."""
486 return os.path.join(self.srcdir, component)
487
488 def component_builddir(self, action, config, component, subconfig=None):
489 """Return the directory to use for a build."""
490 if config is None:
491 # Host libraries.
492 assert subconfig is None
493 return os.path.join(self.builddir, action, component)
494 if subconfig is None:
495 return os.path.join(self.builddir, action, config, component)
496 else:
497 # glibc build as part of compiler build.
498 return os.path.join(self.builddir, action, config, component,
499 subconfig)
500
501 def compiler_installdir(self, config):
502 """Return the directory in which to install a compiler."""
503 return os.path.join(self.installdir, 'compilers', config)
504
505 def compiler_bindir(self, config):
506 """Return the directory in which to find compiler binaries."""
507 return os.path.join(self.compiler_installdir(config), 'bin')
508
509 def compiler_sysroot(self, config):
510 """Return the sysroot directory for a compiler."""
511 return os.path.join(self.compiler_installdir(config), 'sysroot')
512
513 def glibc_installdir(self, config):
514 """Return the directory in which to install glibc."""
515 return os.path.join(self.installdir, 'glibcs', config)
516
517 def run_builds(self, action, configs):
518 """Run the requested builds."""
519 if action == 'checkout':
520 self.checkout(configs)
521 return
4d602bce
JM
522 if action == 'bot-cycle':
523 if configs:
524 print('error: configurations specified for bot-cycle')
525 exit(1)
526 self.bot_cycle()
527 return
a1f6a9ab
JM
528 if action == 'bot':
529 if configs:
530 print('error: configurations specified for bot')
531 exit(1)
532 self.bot()
533 return
feaa1506
FW
534 if action in ('host-libraries', 'list-compilers',
535 'list-glibcs') and configs:
536 print('error: configurations specified for ' + action)
bf469f0c 537 exit(1)
feaa1506
FW
538 if action == 'list-compilers':
539 for name in sorted(self.configs.keys()):
540 print(name)
541 return
542 if action == 'list-glibcs':
543 for config in sorted(self.glibc_configs.values(),
544 key=lambda c: c.name):
545 print(config.name, config.compiler.name)
546 return
bf469f0c
JM
547 self.clear_last_build_state(action)
548 build_time = datetime.datetime.utcnow()
549 if action == 'host-libraries':
550 build_components = ('gmp', 'mpfr', 'mpc')
551 old_components = ()
552 old_versions = {}
14f95a42
JM
553 self.build_host_libraries()
554 elif action == 'compilers':
394b5bac
JM
555 build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
556 'gnumach', 'hurd')
bf469f0c
JM
557 old_components = ('gmp', 'mpfr', 'mpc')
558 old_versions = self.build_state['host-libraries']['build-versions']
14f95a42
JM
559 self.build_compilers(configs)
560 else:
bf469f0c 561 build_components = ('glibc',)
394b5bac
JM
562 old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
563 'mig', 'gnumach', 'hurd')
bf469f0c 564 old_versions = self.build_state['compilers']['build-versions']
07a44d23
FW
565 if action == 'update-syscalls':
566 self.update_syscalls(configs)
567 else:
568 self.build_glibcs(configs)
14f95a42
JM
569 self.write_files()
570 self.do_build()
bf469f0c
JM
571 if configs:
572 # Partial build, do not update stored state.
573 return
574 build_versions = {}
575 for k in build_components:
576 if k in self.versions:
577 build_versions[k] = {'version': self.versions[k]['version'],
578 'revision': self.versions[k]['revision']}
579 for k in old_components:
580 if k in old_versions:
581 build_versions[k] = {'version': old_versions[k]['version'],
582 'revision': old_versions[k]['revision']}
583 self.update_build_state(action, build_time, build_versions)
14f95a42
JM
584
585 @staticmethod
586 def remove_dirs(*args):
587 """Remove directories and their contents if they exist."""
588 for dir in args:
589 shutil.rmtree(dir, ignore_errors=True)
590
591 @staticmethod
592 def remove_recreate_dirs(*args):
593 """Remove directories if they exist, and create them as empty."""
594 Context.remove_dirs(*args)
595 for dir in args:
596 os.makedirs(dir, exist_ok=True)
597
598 def add_makefile_cmdlist(self, target, cmdlist, logsdir):
599 """Add makefile text for a list of commands."""
600 commands = cmdlist.makefile_commands(self.wrapper, logsdir)
601 self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
602 (target, target, target, commands))
bf469f0c 603 self.status_log_list.extend(cmdlist.status_logs(logsdir))
14f95a42
JM
604
605 def write_files(self):
606 """Write out the Makefile and wrapper script."""
607 mftext = ''.join(self.makefile_pieces)
608 with open(self.makefile, 'w') as f:
609 f.write(mftext)
610 wrapper_text = (
611 '#!/bin/sh\n'
612 'prev_base=$1\n'
613 'this_base=$2\n'
614 'desc=$3\n'
615 'dir=$4\n'
616 'path=$5\n'
617 'shift 5\n'
618 'prev_status=$prev_base-status.txt\n'
619 'this_status=$this_base-status.txt\n'
620 'this_log=$this_base-log.txt\n'
621 'date > "$this_log"\n'
622 'echo >> "$this_log"\n'
623 'echo "Description: $desc" >> "$this_log"\n'
8885f979
JM
624 'printf "%s" "Command:" >> "$this_log"\n'
625 'for word in "$@"; do\n'
626 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
627 ' printf " %s" "$word"\n'
628 ' else\n'
629 ' printf " \'"\n'
630 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
631 ' printf "\'"\n'
632 ' fi\n'
633 'done >> "$this_log"\n'
634 'echo >> "$this_log"\n'
14f95a42
JM
635 'echo "Directory: $dir" >> "$this_log"\n'
636 'echo "Path addition: $path" >> "$this_log"\n'
637 'echo >> "$this_log"\n'
638 'record_status ()\n'
639 '{\n'
640 ' echo >> "$this_log"\n'
641 ' echo "$1: $desc" > "$this_status"\n'
642 ' echo "$1: $desc" >> "$this_log"\n'
643 ' echo >> "$this_log"\n'
644 ' date >> "$this_log"\n'
645 ' echo "$1: $desc"\n'
646 ' exit 0\n'
647 '}\n'
648 'check_error ()\n'
649 '{\n'
650 ' if [ "$1" != "0" ]; then\n'
651 ' record_status FAIL\n'
652 ' fi\n'
653 '}\n'
654 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
655 ' record_status UNRESOLVED\n'
656 'fi\n'
657 'if [ "$dir" ]; then\n'
658 ' cd "$dir"\n'
659 ' check_error "$?"\n'
660 'fi\n'
661 'if [ "$path" ]; then\n'
662 ' PATH=$path:$PATH\n'
663 'fi\n'
664 '"$@" < /dev/null >> "$this_log" 2>&1\n'
665 'check_error "$?"\n'
666 'record_status PASS\n')
667 with open(self.wrapper, 'w') as f:
668 f.write(wrapper_text)
0c95f51d
JM
669 # Mode 0o755.
670 mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
671 stat.S_IROTH|stat.S_IXOTH)
672 os.chmod(self.wrapper, mode_exec)
14f95a42
JM
673 save_logs_text = (
674 '#!/bin/sh\n'
675 'if ! [ -f tests.sum ]; then\n'
676 ' echo "No test summary available."\n'
677 ' exit 0\n'
678 'fi\n'
679 'save_file ()\n'
680 '{\n'
681 ' echo "Contents of $1:"\n'
682 ' echo\n'
683 ' cat "$1"\n'
684 ' echo\n'
685 ' echo "End of contents of $1."\n'
686 ' echo\n'
687 '}\n'
688 'save_file tests.sum\n'
689 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
690 'for t in $non_pass_tests; do\n'
691 ' if [ -f "$t.out" ]; then\n'
692 ' save_file "$t.out"\n'
693 ' fi\n'
694 'done\n')
695 with open(self.save_logs, 'w') as f:
696 f.write(save_logs_text)
0c95f51d 697 os.chmod(self.save_logs, mode_exec)
14f95a42
JM
698
699 def do_build(self):
700 """Do the actual build."""
418f5783 701 cmd = ['make', '-O', '-j%d' % self.parallelism]
14f95a42
JM
702 subprocess.run(cmd, cwd=self.builddir, check=True)
703
704 def build_host_libraries(self):
705 """Build the host libraries."""
706 installdir = self.host_libraries_installdir
707 builddir = os.path.join(self.builddir, 'host-libraries')
708 logsdir = os.path.join(self.logsdir, 'host-libraries')
709 self.remove_recreate_dirs(installdir, builddir, logsdir)
710 cmdlist = CommandList('host-libraries', self.keep)
711 self.build_host_library(cmdlist, 'gmp')
712 self.build_host_library(cmdlist, 'mpfr',
713 ['--with-gmp=%s' % installdir])
714 self.build_host_library(cmdlist, 'mpc',
715 ['--with-gmp=%s' % installdir,
716 '--with-mpfr=%s' % installdir])
717 cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
718 self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
719
720 def build_host_library(self, cmdlist, lib, extra_opts=None):
721 """Build one host library."""
722 srcdir = self.component_srcdir(lib)
723 builddir = self.component_builddir('host-libraries', None, lib)
724 installdir = self.host_libraries_installdir
725 cmdlist.push_subdesc(lib)
726 cmdlist.create_use_dir(builddir)
727 cfg_cmd = [os.path.join(srcdir, 'configure'),
728 '--prefix=%s' % installdir,
729 '--disable-shared']
730 if extra_opts:
731 cfg_cmd.extend (extra_opts)
732 cmdlist.add_command('configure', cfg_cmd)
733 cmdlist.add_command('build', ['make'])
734 cmdlist.add_command('check', ['make', 'check'])
735 cmdlist.add_command('install', ['make', 'install'])
736 cmdlist.cleanup_dir()
737 cmdlist.pop_subdesc()
738
739 def build_compilers(self, configs):
740 """Build the compilers."""
741 if not configs:
742 self.remove_dirs(os.path.join(self.builddir, 'compilers'))
743 self.remove_dirs(os.path.join(self.installdir, 'compilers'))
744 self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
745 configs = sorted(self.configs.keys())
746 for c in configs:
747 self.configs[c].build()
748
749 def build_glibcs(self, configs):
750 """Build the glibcs."""
751 if not configs:
752 self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
753 self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
754 self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
755 configs = sorted(self.glibc_configs.keys())
756 for c in configs:
757 self.glibc_configs[c].build()
758
07a44d23
FW
759 def update_syscalls(self, configs):
760 """Update the glibc syscall lists."""
761 if not configs:
762 self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
763 self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
764 configs = sorted(self.glibc_configs.keys())
765 for c in configs:
766 self.glibc_configs[c].update_syscalls()
767
02c78f02
JM
768 def load_versions_json(self):
769 """Load information about source directory versions."""
770 if not os.access(self.versions_json, os.F_OK):
771 self.versions = {}
772 return
773 with open(self.versions_json, 'r') as f:
774 self.versions = json.load(f)
775
776 def store_json(self, data, filename):
777 """Store information in a JSON file."""
778 filename_tmp = filename + '.tmp'
779 with open(filename_tmp, 'w') as f:
780 json.dump(data, f, indent=2, sort_keys=True)
781 os.rename(filename_tmp, filename)
782
783 def store_versions_json(self):
784 """Store information about source directory versions."""
785 self.store_json(self.versions, self.versions_json)
786
787 def set_component_version(self, component, version, explicit, revision):
788 """Set the version information for a component."""
789 self.versions[component] = {'version': version,
790 'explicit': explicit,
791 'revision': revision}
792 self.store_versions_json()
793
14f95a42
JM
794 def checkout(self, versions):
795 """Check out the desired component versions."""
9125e43d 796 default_versions = {'binutils': 'vcs-2.39',
c33ada06 797 'gcc': 'vcs-12',
14f95a42 798 'glibc': 'vcs-mainline',
7366cb03 799 'gmp': '6.2.1',
1f3f0060 800 'linux': '6.0',
b853a417 801 'mpc': '1.2.1',
63ce2425 802 'mpfr': '4.1.0',
f4d3cee3 803 'mig': 'vcs-mainline',
19fd8d0d 804 'gnumach': 'vcs-mainline',
47c4b4b0 805 'hurd': 'vcs-mainline'}
14f95a42 806 use_versions = {}
02c78f02 807 explicit_versions = {}
14f95a42
JM
808 for v in versions:
809 found_v = False
810 for k in default_versions.keys():
811 kx = k + '-'
812 if v.startswith(kx):
813 vx = v[len(kx):]
814 if k in use_versions:
815 print('error: multiple versions for %s' % k)
816 exit(1)
817 use_versions[k] = vx
02c78f02 818 explicit_versions[k] = True
14f95a42
JM
819 found_v = True
820 break
821 if not found_v:
822 print('error: unknown component in %s' % v)
823 exit(1)
824 for k in default_versions.keys():
825 if k not in use_versions:
02c78f02
JM
826 if k in self.versions and self.versions[k]['explicit']:
827 use_versions[k] = self.versions[k]['version']
828 explicit_versions[k] = True
829 else:
830 use_versions[k] = default_versions[k]
831 explicit_versions[k] = False
14f95a42
JM
832 os.makedirs(self.srcdir, exist_ok=True)
833 for k in sorted(default_versions.keys()):
834 update = os.access(self.component_srcdir(k), os.F_OK)
835 v = use_versions[k]
02c78f02
JM
836 if (update and
837 k in self.versions and
838 v != self.versions[k]['version']):
839 if not self.replace_sources:
840 print('error: version of %s has changed from %s to %s, '
841 'use --replace-sources to check out again' %
842 (k, self.versions[k]['version'], v))
843 exit(1)
844 shutil.rmtree(self.component_srcdir(k))
845 update = False
14f95a42 846 if v.startswith('vcs-'):
02c78f02 847 revision = self.checkout_vcs(k, v[4:], update)
14f95a42
JM
848 else:
849 self.checkout_tar(k, v, update)
02c78f02
JM
850 revision = v
851 self.set_component_version(k, v, explicit_versions[k], revision)
a1c9859b
JM
852 if self.get_script_text() != self.script_text:
853 # Rerun the checkout process in case the updated script
854 # uses different default versions or new components.
855 self.exec_self()
14f95a42
JM
856
857 def checkout_vcs(self, component, version, update):
858 """Check out the given version of the given component from version
02c78f02 859 control. Return a revision identifier."""
14f95a42 860 if component == 'binutils':
0784e791 861 git_url = 'https://sourceware.org/git/binutils-gdb.git'
14f95a42
JM
862 if version == 'mainline':
863 git_branch = 'master'
864 else:
865 trans = str.maketrans({'.': '_'})
866 git_branch = 'binutils-%s-branch' % version.translate(trans)
02c78f02 867 return self.git_checkout(component, git_url, git_branch, update)
14f95a42
JM
868 elif component == 'gcc':
869 if version == 'mainline':
bc487d71 870 branch = 'master'
14f95a42 871 else:
bc487d71
JM
872 branch = 'releases/gcc-%s' % version
873 return self.gcc_checkout(branch, update)
14f95a42 874 elif component == 'glibc':
0784e791 875 git_url = 'https://sourceware.org/git/glibc.git'
14f95a42
JM
876 if version == 'mainline':
877 git_branch = 'master'
878 else:
879 git_branch = 'release/%s/master' % version
02c78f02 880 r = self.git_checkout(component, git_url, git_branch, update)
14f95a42 881 self.fix_glibc_timestamps()
02c78f02 882 return r
19fd8d0d
ST
883 elif component == 'gnumach':
884 git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
885 git_branch = 'master'
886 r = self.git_checkout(component, git_url, git_branch, update)
887 subprocess.run(['autoreconf', '-i'],
888 cwd=self.component_srcdir(component), check=True)
889 return r
f4d3cee3
ST
890 elif component == 'mig':
891 git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
892 git_branch = 'master'
893 r = self.git_checkout(component, git_url, git_branch, update)
894 subprocess.run(['autoreconf', '-i'],
895 cwd=self.component_srcdir(component), check=True)
896 return r
f9015003
ST
897 elif component == 'hurd':
898 git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
899 git_branch = 'master'
48130690
ST
900 r = self.git_checkout(component, git_url, git_branch, update)
901 subprocess.run(['autoconf'],
902 cwd=self.component_srcdir(component), check=True)
903 return r
14f95a42
JM
904 else:
905 print('error: component %s coming from VCS' % component)
906 exit(1)
907
908 def git_checkout(self, component, git_url, git_branch, update):
02c78f02 909 """Check out a component from git. Return a commit identifier."""
14f95a42
JM
910 if update:
911 subprocess.run(['git', 'remote', 'prune', 'origin'],
912 cwd=self.component_srcdir(component), check=True)
e3aa6999
JM
913 if self.replace_sources:
914 subprocess.run(['git', 'clean', '-dxfq'],
915 cwd=self.component_srcdir(component), check=True)
14f95a42
JM
916 subprocess.run(['git', 'pull', '-q'],
917 cwd=self.component_srcdir(component), check=True)
918 else:
c592721a
FW
919 if self.shallow:
920 depth_arg = ('--depth', '1')
921 else:
922 depth_arg = ()
923 subprocess.run(['git', 'clone', '-q', '-b', git_branch,
924 *depth_arg, git_url,
14f95a42 925 self.component_srcdir(component)], check=True)
02c78f02
JM
926 r = subprocess.run(['git', 'rev-parse', 'HEAD'],
927 cwd=self.component_srcdir(component),
928 stdout=subprocess.PIPE,
929 check=True, universal_newlines=True).stdout
930 return r.rstrip()
14f95a42
JM
931
932 def fix_glibc_timestamps(self):
933 """Fix timestamps in a glibc checkout."""
934 # Ensure that builds do not try to regenerate generated files
935 # in the source tree.
936 srcdir = self.component_srcdir('glibc')
2bd0bfcc
JM
937 # These files have Makefile dependencies to regenerate them in
938 # the source tree that may be active during a normal build.
939 # Some other files have such dependencies but do not need to
940 # be touched because nothing in a build depends on the files
941 # in question.
f13d2601 942 for f in ('sysdeps/mach/hurd/bits/errno.h',):
2bd0bfcc
JM
943 to_touch = os.path.join(srcdir, f)
944 subprocess.run(['touch', '-c', to_touch], check=True)
14f95a42
JM
945 for dirpath, dirnames, filenames in os.walk(srcdir):
946 for f in filenames:
947 if (f == 'configure' or
948 f == 'preconfigure' or
949 f.endswith('-kw.h')):
950 to_touch = os.path.join(dirpath, f)
951 subprocess.run(['touch', to_touch], check=True)
952
bc487d71
JM
953 def gcc_checkout(self, branch, update):
954 """Check out GCC from git. Return the commit identifier."""
955 if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
956 os.F_OK):
957 if not self.replace_sources:
958 print('error: GCC has moved from SVN to git, use '
959 '--replace-sources to check out again')
960 exit(1)
961 shutil.rmtree(self.component_srcdir('gcc'))
962 update = False
14f95a42 963 if not update:
0784e791 964 self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
bc487d71 965 branch, update)
14f95a42
JM
966 subprocess.run(['contrib/gcc_update', '--silent'],
967 cwd=self.component_srcdir('gcc'), check=True)
bc487d71
JM
968 r = subprocess.run(['git', 'rev-parse', 'HEAD'],
969 cwd=self.component_srcdir('gcc'),
02c78f02
JM
970 stdout=subprocess.PIPE,
971 check=True, universal_newlines=True).stdout
972 return r.rstrip()
14f95a42
JM
973
974 def checkout_tar(self, component, version, update):
975 """Check out the given version of the given component from a
976 tarball."""
977 if update:
978 return
979 url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
c8fabb84 980 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
14f95a42 981 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
0c1041ee 982 'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
14f95a42 983 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
394b5bac
JM
984 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
985 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
986 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
987 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
14f95a42
JM
988 if component not in url_map:
989 print('error: component %s coming from tarball' % component)
990 exit(1)
0c1041ee
JM
991 version_major = version.split('.')[0]
992 url = url_map[component] % {'version': version, 'major': version_major}
14f95a42
JM
993 filename = os.path.join(self.srcdir, url.split('/')[-1])
994 response = urllib.request.urlopen(url)
995 data = response.read()
996 with open(filename, 'wb') as f:
997 f.write(data)
998 subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
999 check=True)
1000 os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1001 self.component_srcdir(component))
1002 os.remove(filename)
1003
bf469f0c
JM
1004 def load_build_state_json(self):
1005 """Load information about the state of previous builds."""
1006 if os.access(self.build_state_json, os.F_OK):
1007 with open(self.build_state_json, 'r') as f:
1008 self.build_state = json.load(f)
1009 else:
1010 self.build_state = {}
07a44d23 1011 for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
bf469f0c
JM
1012 if k not in self.build_state:
1013 self.build_state[k] = {}
1014 if 'build-time' not in self.build_state[k]:
1015 self.build_state[k]['build-time'] = ''
1016 if 'build-versions' not in self.build_state[k]:
1017 self.build_state[k]['build-versions'] = {}
1018 if 'build-results' not in self.build_state[k]:
1019 self.build_state[k]['build-results'] = {}
1020 if 'result-changes' not in self.build_state[k]:
1021 self.build_state[k]['result-changes'] = {}
1022 if 'ever-passed' not in self.build_state[k]:
1023 self.build_state[k]['ever-passed'] = []
1024
1025 def store_build_state_json(self):
1026 """Store information about the state of previous builds."""
1027 self.store_json(self.build_state, self.build_state_json)
1028
1029 def clear_last_build_state(self, action):
1030 """Clear information about the state of part of the build."""
1031 # We clear the last build time and versions when starting a
1032 # new build. The results of the last build are kept around,
1033 # as comparison is still meaningful if this build is aborted
1034 # and a new one started.
1035 self.build_state[action]['build-time'] = ''
1036 self.build_state[action]['build-versions'] = {}
1037 self.store_build_state_json()
1038
1039 def update_build_state(self, action, build_time, build_versions):
1040 """Update the build state after a build."""
1041 build_time = build_time.replace(microsecond=0)
1042 self.build_state[action]['build-time'] = str(build_time)
1043 self.build_state[action]['build-versions'] = build_versions
1044 build_results = {}
1045 for log in self.status_log_list:
1046 with open(log, 'r') as f:
1047 log_text = f.read()
1048 log_text = log_text.rstrip()
1049 m = re.fullmatch('([A-Z]+): (.*)', log_text)
1050 result = m.group(1)
1051 test_name = m.group(2)
1052 assert test_name not in build_results
1053 build_results[test_name] = result
1054 old_build_results = self.build_state[action]['build-results']
1055 self.build_state[action]['build-results'] = build_results
1056 result_changes = {}
1057 all_tests = set(old_build_results.keys()) | set(build_results.keys())
1058 for t in all_tests:
1059 if t in old_build_results:
1060 old_res = old_build_results[t]
1061 else:
1062 old_res = '(New test)'
1063 if t in build_results:
1064 new_res = build_results[t]
1065 else:
1066 new_res = '(Test removed)'
1067 if old_res != new_res:
1068 result_changes[t] = '%s -> %s' % (old_res, new_res)
1069 self.build_state[action]['result-changes'] = result_changes
1070 old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1071 if t in build_results}
1072 new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1073 self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1074 new_passes)
1075 self.store_build_state_json()
1076
4d602bce
JM
1077 def load_bot_config_json(self):
1078 """Load bot configuration."""
1079 with open(self.bot_config_json, 'r') as f:
1080 self.bot_config = json.load(f)
1081
1082 def part_build_old(self, action, delay):
1083 """Return whether the last build for a given action was at least a
1084 given number of seconds ago, or does not have a time recorded."""
1085 old_time_str = self.build_state[action]['build-time']
1086 if not old_time_str:
1087 return True
1088 old_time = datetime.datetime.strptime(old_time_str,
1089 '%Y-%m-%d %H:%M:%S')
1090 new_time = datetime.datetime.utcnow()
1091 delta = new_time - old_time
1092 return delta.total_seconds() >= delay
1093
1094 def bot_cycle(self):
1095 """Run a single round of checkout and builds."""
1096 print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1097 self.load_bot_config_json()
1098 actions = ('host-libraries', 'compilers', 'glibcs')
1099 self.bot_run_self(['--replace-sources'], 'checkout')
1100 self.load_versions_json()
1101 if self.get_script_text() != self.script_text:
1102 print('Script changed, re-execing.')
1103 # On script change, all parts of the build should be rerun.
1104 for a in actions:
1105 self.clear_last_build_state(a)
1106 self.exec_self()
1107 check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
394b5bac
JM
1108 'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1109 'mig', 'gnumach', 'hurd'),
4d602bce
JM
1110 'glibcs': ('glibc',)}
1111 must_build = {}
1112 for a in actions:
1113 build_vers = self.build_state[a]['build-versions']
1114 must_build[a] = False
1115 if not self.build_state[a]['build-time']:
1116 must_build[a] = True
1117 old_vers = {}
1118 new_vers = {}
1119 for c in check_components[a]:
1120 if c in build_vers:
1121 old_vers[c] = build_vers[c]
1122 new_vers[c] = {'version': self.versions[c]['version'],
1123 'revision': self.versions[c]['revision']}
1124 if new_vers == old_vers:
1125 print('Versions for %s unchanged.' % a)
1126 else:
1127 print('Versions changed or rebuild forced for %s.' % a)
1128 if a == 'compilers' and not self.part_build_old(
1129 a, self.bot_config['compilers-rebuild-delay']):
1130 print('Not requiring rebuild of compilers this soon.')
1131 else:
1132 must_build[a] = True
1133 if must_build['host-libraries']:
1134 must_build['compilers'] = True
1135 if must_build['compilers']:
1136 must_build['glibcs'] = True
1137 for a in actions:
1138 if must_build[a]:
1139 print('Must rebuild %s.' % a)
1140 self.clear_last_build_state(a)
1141 else:
1142 print('No need to rebuild %s.' % a)
a1f6a9ab
JM
1143 if os.access(self.logsdir, os.F_OK):
1144 shutil.rmtree(self.logsdir_old, ignore_errors=True)
1145 shutil.copytree(self.logsdir, self.logsdir_old)
4d602bce
JM
1146 for a in actions:
1147 if must_build[a]:
1148 build_time = datetime.datetime.utcnow()
1149 print('Rebuilding %s at %s.' % (a, str(build_time)))
1150 self.bot_run_self([], a)
1151 self.load_build_state_json()
1152 self.bot_build_mail(a, build_time)
1153 print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1154
1155 def bot_build_mail(self, action, build_time):
1156 """Send email with the results of a build."""
f0166c16
ZW
1157 if not ('email-from' in self.bot_config and
1158 'email-server' in self.bot_config and
1159 'email-subject' in self.bot_config and
1160 'email-to' in self.bot_config):
1161 if not self.email_warning:
1162 print("Email not configured, not sending.")
1163 self.email_warning = True
1164 return
1165
4d602bce
JM
1166 build_time = build_time.replace(microsecond=0)
1167 subject = (self.bot_config['email-subject'] %
1168 {'action': action,
1169 'build-time': str(build_time)})
1170 results = self.build_state[action]['build-results']
1171 changes = self.build_state[action]['result-changes']
1172 ever_passed = set(self.build_state[action]['ever-passed'])
1173 versions = self.build_state[action]['build-versions']
1174 new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1175 all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1176 all_fails = {k for k in results if results[k] == 'FAIL'}
1177 if new_regressions:
1178 new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1179 new_reg_text = ('New regressions:\n\n%s\n\n' %
1180 '\n'.join(new_reg_list))
1181 else:
1182 new_reg_text = ''
1183 if all_regressions:
1184 all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1185 all_reg_text = ('All regressions:\n\n%s\n\n' %
1186 '\n'.join(all_reg_list))
1187 else:
1188 all_reg_text = ''
1189 if all_fails:
1190 all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1191 all_fail_text = ('All failures:\n\n%s\n\n' %
1192 '\n'.join(all_fail_list))
1193 else:
1194 all_fail_text = ''
1195 if changes:
1196 changes_list = sorted(changes.keys())
1197 changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1198 changes_text = ('All changed results:\n\n%s\n\n' %
1199 '\n'.join(changes_list))
1200 else:
1201 changes_text = ''
1202 results_text = (new_reg_text + all_reg_text + all_fail_text +
1203 changes_text)
1204 if not results_text:
1205 results_text = 'Clean build with unchanged results.\n\n'
1206 versions_list = sorted(versions.keys())
1207 versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1208 versions[k]['revision'])
1209 for k in versions_list]
1210 versions_text = ('Component versions for this build:\n\n%s\n' %
1211 '\n'.join(versions_list))
1212 body_text = results_text + versions_text
1213 msg = email.mime.text.MIMEText(body_text)
1214 msg['Subject'] = subject
1215 msg['From'] = self.bot_config['email-from']
1216 msg['To'] = self.bot_config['email-to']
1217 msg['Message-ID'] = email.utils.make_msgid()
1218 msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1219 with smtplib.SMTP(self.bot_config['email-server']) as s:
1220 s.send_message(msg)
1221
a1f6a9ab 1222 def bot_run_self(self, opts, action, check=True):
4d602bce
JM
1223 """Run a copy of this script with given options."""
1224 cmd = [sys.executable, sys.argv[0], '--keep=none',
1225 '-j%d' % self.parallelism]
b1176270
JM
1226 if self.full_gcc:
1227 cmd.append('--full-gcc')
4d602bce
JM
1228 cmd.extend(opts)
1229 cmd.extend([self.topdir, action])
a1f6a9ab
JM
1230 sys.stdout.flush()
1231 subprocess.run(cmd, check=check)
1232
1233 def bot(self):
1234 """Run repeated rounds of checkout and builds."""
1235 while True:
1236 self.load_bot_config_json()
1237 if not self.bot_config['run']:
1238 print('Bot exiting by request.')
1239 exit(0)
1240 self.bot_run_self([], 'bot-cycle', check=False)
1241 self.load_bot_config_json()
1242 if not self.bot_config['run']:
1243 print('Bot exiting by request.')
1244 exit(0)
1245 time.sleep(self.bot_config['delay'])
1246 if self.get_script_text() != self.script_text:
1247 print('Script changed, bot re-execing.')
1248 self.exec_self()
4d602bce 1249
65b6c9b0
FW
1250class LinuxHeadersPolicyForBuild(object):
1251 """Names and directories for installing Linux headers. Build variant."""
1252
1253 def __init__(self, config):
1254 self.arch = config.arch
1255 self.srcdir = config.ctx.component_srcdir('linux')
1256 self.builddir = config.component_builddir('linux')
1257 self.headers_dir = os.path.join(config.sysroot, 'usr')
1258
07a44d23
FW
1259class LinuxHeadersPolicyForUpdateSyscalls(object):
1260 """Names and directories for Linux headers. update-syscalls variant."""
1261
1262 def __init__(self, glibc, headers_dir):
1263 self.arch = glibc.compiler.arch
1264 self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1265 self.builddir = glibc.ctx.component_builddir(
1266 'update-syscalls', glibc.name, 'build-linux')
1267 self.headers_dir = headers_dir
1268
65b6c9b0
FW
1269def install_linux_headers(policy, cmdlist):
1270 """Install Linux kernel headers."""
1271 arch_map = {'aarch64': 'arm64',
1272 'alpha': 'alpha',
2fc2260b 1273 'arc': 'arc',
65b6c9b0
FW
1274 'arm': 'arm',
1275 'csky': 'csky',
1276 'hppa': 'parisc',
1277 'i486': 'x86',
1278 'i586': 'x86',
1279 'i686': 'x86',
1280 'i786': 'x86',
1281 'ia64': 'ia64',
3be5fc00 1282 'loongarch64': 'loongarch',
65b6c9b0
FW
1283 'm68k': 'm68k',
1284 'microblaze': 'microblaze',
1285 'mips': 'mips',
1286 'nios2': 'nios2',
e9816506 1287 'or1k': 'openrisc',
65b6c9b0
FW
1288 'powerpc': 'powerpc',
1289 's390': 's390',
1290 'riscv32': 'riscv',
1291 'riscv64': 'riscv',
1292 'sh': 'sh',
1293 'sparc': 'sparc',
1294 'x86_64': 'x86'}
1295 linux_arch = None
1296 for k in arch_map:
1297 if policy.arch.startswith(k):
1298 linux_arch = arch_map[k]
1299 break
1300 assert linux_arch is not None
1301 cmdlist.push_subdesc('linux')
1302 cmdlist.create_use_dir(policy.builddir)
1303 cmdlist.add_command('install-headers',
1304 ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1305 'ARCH=%s' % linux_arch,
1306 'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1307 'headers_install'])
1308 cmdlist.cleanup_dir()
1309 cmdlist.pop_subdesc()
14f95a42 1310
0c95f51d 1311class Config(object):
14f95a42
JM
1312 """A configuration for building a compiler and associated libraries."""
1313
1314 def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
23a7896d
JM
1315 first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1316 extra_glibcs=None):
14f95a42
JM
1317 """Initialize a Config object."""
1318 self.ctx = ctx
1319 self.arch = arch
1320 self.os = os_name
1321 self.variant = variant
1322 if variant is None:
1323 self.name = '%s-%s' % (arch, os_name)
1324 else:
1325 self.name = '%s-%s-%s' % (arch, os_name, variant)
1326 self.triplet = '%s-glibc-%s' % (arch, os_name)
1327 if gcc_cfg is None:
1328 self.gcc_cfg = []
1329 else:
1330 self.gcc_cfg = gcc_cfg
1331 if first_gcc_cfg is None:
1332 self.first_gcc_cfg = []
1333 else:
1334 self.first_gcc_cfg = first_gcc_cfg
23a7896d
JM
1335 if binutils_cfg is None:
1336 self.binutils_cfg = []
1337 else:
1338 self.binutils_cfg = binutils_cfg
14f95a42
JM
1339 if glibcs is None:
1340 glibcs = [{'variant': variant}]
1341 if extra_glibcs is None:
1342 extra_glibcs = []
1343 glibcs = [Glibc(self, **g) for g in glibcs]
1344 extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1345 self.all_glibcs = glibcs + extra_glibcs
1346 self.compiler_glibcs = glibcs
1347 self.installdir = ctx.compiler_installdir(self.name)
1348 self.bindir = ctx.compiler_bindir(self.name)
1349 self.sysroot = ctx.compiler_sysroot(self.name)
1350 self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1351 self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1352
1353 def component_builddir(self, component):
1354 """Return the directory to use for a (non-glibc) build."""
1355 return self.ctx.component_builddir('compilers', self.name, component)
1356
1357 def build(self):
1358 """Generate commands to build this compiler."""
1359 self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1360 self.logsdir)
1361 cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1362 cmdlist.add_command('check-host-libraries',
1363 ['test', '-f',
1364 os.path.join(self.ctx.host_libraries_installdir,
1365 'ok')])
1366 cmdlist.use_path(self.bindir)
1367 self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1368 ['--disable-gdb',
a179673f 1369 '--disable-gdbserver',
14f95a42
JM
1370 '--disable-libdecnumber',
1371 '--disable-readline',
23a7896d 1372 '--disable-sim'] + self.binutils_cfg)
14f95a42 1373 if self.os.startswith('linux'):
65b6c9b0 1374 install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
14f95a42 1375 self.build_gcc(cmdlist, True)
394b5bac
JM
1376 if self.os == 'gnu':
1377 self.install_gnumach_headers(cmdlist)
1378 self.build_cross_tool(cmdlist, 'mig', 'mig')
1379 self.install_hurd_headers(cmdlist)
14f95a42
JM
1380 for g in self.compiler_glibcs:
1381 cmdlist.push_subdesc('glibc')
1382 cmdlist.push_subdesc(g.name)
857c7d73 1383 g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
14f95a42
JM
1384 cmdlist.pop_subdesc()
1385 cmdlist.pop_subdesc()
1386 self.build_gcc(cmdlist, False)
1387 cmdlist.add_command('done', ['touch',
1388 os.path.join(self.installdir, 'ok')])
1389 self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1390 self.logsdir)
1391
1392 def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1393 """Build one cross tool."""
1394 srcdir = self.ctx.component_srcdir(tool_src)
1395 builddir = self.component_builddir(tool_build)
1396 cmdlist.push_subdesc(tool_build)
1397 cmdlist.create_use_dir(builddir)
1398 cfg_cmd = [os.path.join(srcdir, 'configure'),
1399 '--prefix=%s' % self.installdir,
1400 '--build=%s' % self.ctx.build_triplet,
1401 '--host=%s' % self.ctx.build_triplet,
1402 '--target=%s' % self.triplet,
1403 '--with-sysroot=%s' % self.sysroot]
1404 if extra_opts:
1405 cfg_cmd.extend(extra_opts)
1406 cmdlist.add_command('configure', cfg_cmd)
1407 cmdlist.add_command('build', ['make'])
cd880aa2
JM
1408 # Parallel "make install" for GCC has race conditions that can
1409 # cause it to fail; see
1410 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such
1411 # problems are not known for binutils, but doing the
1412 # installation in parallel within a particular toolchain build
1413 # (as opposed to installation of one toolchain from
1414 # build-many-glibcs.py running in parallel to the installation
1415 # of other toolchains being built) is not known to be
1416 # significantly beneficial, so it is simplest just to disable
1417 # parallel install for cross tools here.
1418 cmdlist.add_command('install', ['make', '-j1', 'install'])
14f95a42
JM
1419 cmdlist.cleanup_dir()
1420 cmdlist.pop_subdesc()
1421
394b5bac
JM
1422 def install_gnumach_headers(self, cmdlist):
1423 """Install GNU Mach headers."""
1424 srcdir = self.ctx.component_srcdir('gnumach')
1425 builddir = self.component_builddir('gnumach')
1426 cmdlist.push_subdesc('gnumach')
1427 cmdlist.create_use_dir(builddir)
1428 cmdlist.add_command('configure',
1429 [os.path.join(srcdir, 'configure'),
1430 '--build=%s' % self.ctx.build_triplet,
1431 '--host=%s' % self.triplet,
1432 '--prefix=',
1433 'CC=%s-gcc -nostdlib' % self.triplet])
1434 cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1435 'install-data'])
1436 cmdlist.cleanup_dir()
1437 cmdlist.pop_subdesc()
1438
1439 def install_hurd_headers(self, cmdlist):
1440 """Install Hurd headers."""
1441 srcdir = self.ctx.component_srcdir('hurd')
1442 builddir = self.component_builddir('hurd')
1443 cmdlist.push_subdesc('hurd')
1444 cmdlist.create_use_dir(builddir)
1445 cmdlist.add_command('configure',
1446 [os.path.join(srcdir, 'configure'),
1447 '--build=%s' % self.ctx.build_triplet,
1448 '--host=%s' % self.triplet,
1449 '--prefix=',
1450 '--disable-profile', '--without-parted',
1451 'CC=%s-gcc -nostdlib' % self.triplet])
1452 cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1453 'no_deps=t', 'install-headers'])
1454 cmdlist.cleanup_dir()
1455 cmdlist.pop_subdesc()
1456
14f95a42
JM
1457 def build_gcc(self, cmdlist, bootstrap):
1458 """Build GCC."""
b1176270
JM
1459 # libssp is of little relevance with glibc's own stack
1460 # checking support. libcilkrts does not support GNU/Hurd (and
1461 # has been removed in GCC 8, so --disable-libcilkrts can be
1462 # removed once glibc no longer supports building with older
cbcd65c8
L
1463 # GCC versions). --enable-initfini-array is enabled by default
1464 # in GCC 12, which can be removed when GCC 12 becomes the
1465 # minimum requirement.
14f95a42 1466 cfg_opts = list(self.gcc_cfg)
a586fe9c 1467 cfg_opts += ['--enable-initfini-array']
b1176270 1468 cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
c440d5d5
JM
1469 host_libs = self.ctx.host_libraries_installdir
1470 cfg_opts += ['--with-gmp=%s' % host_libs,
1471 '--with-mpfr=%s' % host_libs,
1472 '--with-mpc=%s' % host_libs]
14f95a42
JM
1473 if bootstrap:
1474 tool_build = 'gcc-first'
1475 # Building a static-only, C-only compiler that is
1476 # sufficient to build glibc. Various libraries and
1477 # features that may require libc headers must be disabled.
1478 # When configuring with a sysroot, --with-newlib is
1479 # required to define inhibit_libc (to stop some parts of
1480 # libgcc including libc headers); --without-headers is not
1481 # sufficient.
1482 cfg_opts += ['--enable-languages=c', '--disable-shared',
1483 '--disable-threads',
1484 '--disable-libatomic',
1485 '--disable-decimal-float',
1486 '--disable-libffi',
1487 '--disable-libgomp',
1488 '--disable-libitm',
1489 '--disable-libmpx',
1490 '--disable-libquadmath',
b1176270 1491 '--disable-libsanitizer',
14f95a42
JM
1492 '--without-headers', '--with-newlib',
1493 '--with-glibc-version=%s' % self.ctx.glibc_version
1494 ]
1495 cfg_opts += self.first_gcc_cfg
1496 else:
1497 tool_build = 'gcc'
b1176270 1498 # libsanitizer commonly breaks because of glibc header
3aec0c39
FW
1499 # changes, or on unusual targets. C++ pre-compiled
1500 # headers are not used during the glibc build and are
1501 # expensive to create.
b1176270 1502 if not self.ctx.full_gcc:
3aec0c39
FW
1503 cfg_opts += ['--disable-libsanitizer',
1504 '--disable-libstdcxx-pch']
b1176270
JM
1505 langs = 'all' if self.ctx.full_gcc else 'c,c++'
1506 cfg_opts += ['--enable-languages=%s' % langs,
1507 '--enable-shared', '--enable-threads']
14f95a42
JM
1508 self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1509
857c7d73
FW
1510class GlibcPolicyDefault(object):
1511 """Build policy for glibc: common defaults."""
1512
1513 def __init__(self, glibc):
1514 self.srcdir = glibc.ctx.component_srcdir('glibc')
1515 self.use_usr = glibc.os != 'gnu'
1516 self.prefix = '/usr' if self.use_usr else ''
1517 self.configure_args = [
1518 '--prefix=%s' % self.prefix,
1519 '--enable-profile',
1520 '--build=%s' % glibc.ctx.build_triplet,
1521 '--host=%s' % glibc.triplet,
1522 'CC=%s' % glibc.tool_name('gcc'),
1523 'CXX=%s' % glibc.tool_name('g++'),
1524 'AR=%s' % glibc.tool_name('ar'),
1525 'AS=%s' % glibc.tool_name('as'),
1526 'LD=%s' % glibc.tool_name('ld'),
1527 'NM=%s' % glibc.tool_name('nm'),
1528 'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1529 'OBJDUMP=%s' % glibc.tool_name('objdump'),
1530 'RANLIB=%s' % glibc.tool_name('ranlib'),
1531 'READELF=%s' % glibc.tool_name('readelf'),
1532 'STRIP=%s' % glibc.tool_name('strip'),
1533 ]
1534 if glibc.os == 'gnu':
1535 self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
33dd32fd
SL
1536 if glibc.cflags:
1537 self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1538 self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
857c7d73
FW
1539 self.configure_args += glibc.cfg
1540
1541 def configure(self, cmdlist):
1542 """Invoked to add the configure command to the command list."""
1543 cmdlist.add_command('configure',
1544 [os.path.join(self.srcdir, 'configure'),
1545 *self.configure_args])
1546
1547 def extra_commands(self, cmdlist):
1548 """Invoked to inject additional commands (make check) after build."""
1549 pass
1550
1551class GlibcPolicyForCompiler(GlibcPolicyDefault):
1552 """Build policy for glibc during the compilers stage."""
1553
1554 def __init__(self, glibc):
1555 super().__init__(glibc)
1556 self.builddir = glibc.ctx.component_builddir(
1557 'compilers', glibc.compiler.name, 'glibc', glibc.name)
1558 self.installdir = glibc.compiler.sysroot
1559
1560class GlibcPolicyForBuild(GlibcPolicyDefault):
1561 """Build policy for glibc during the glibcs stage."""
1562
1563 def __init__(self, glibc):
1564 super().__init__(glibc)
1565 self.builddir = glibc.ctx.component_builddir(
1566 'glibcs', glibc.name, 'glibc')
1567 self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1568 if glibc.ctx.strip:
1569 self.strip = glibc.tool_name('strip')
1570 else:
1571 self.strip = None
1572 self.save_logs = glibc.ctx.save_logs
1573
1574 def extra_commands(self, cmdlist):
1575 if self.strip:
876e5154
JM
1576 # Avoid stripping libc.so and libpthread.so, which are
1577 # linker scripts stored in /lib on Hurd.
1578 find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1579 cmdlist.add_command('strip', ['sh', '-c', (
1580 'set -e; for f in $(%s); do '
1581 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1582 'done' % (find_command, self.strip))])
857c7d73
FW
1583 cmdlist.add_command('check', ['make', 'check'])
1584 cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1585
07a44d23
FW
1586class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1587 """Build policy for glibc during update-syscalls."""
1588
1589 def __init__(self, glibc):
1590 super().__init__(glibc)
1591 self.builddir = glibc.ctx.component_builddir(
1592 'update-syscalls', glibc.name, 'glibc')
1593 self.linuxdir = glibc.ctx.component_builddir(
1594 'update-syscalls', glibc.name, 'linux')
1595 self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1596 glibc, self.linuxdir)
1597 self.configure_args.insert(
1598 0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1599 # self.installdir not set because installation is not supported
14f95a42 1600
0c95f51d 1601class Glibc(object):
14f95a42
JM
1602 """A configuration for building glibc."""
1603
1604 def __init__(self, compiler, arch=None, os_name=None, variant=None,
33dd32fd 1605 cfg=None, ccopts=None, cflags=None):
14f95a42
JM
1606 """Initialize a Glibc object."""
1607 self.ctx = compiler.ctx
1608 self.compiler = compiler
1609 if arch is None:
1610 self.arch = compiler.arch
1611 else:
1612 self.arch = arch
1613 if os_name is None:
1614 self.os = compiler.os
1615 else:
1616 self.os = os_name
1617 self.variant = variant
1618 if variant is None:
1619 self.name = '%s-%s' % (self.arch, self.os)
1620 else:
1621 self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1622 self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1623 if cfg is None:
1624 self.cfg = []
1625 else:
1626 self.cfg = cfg
33dd32fd 1627 # ccopts contain ABI options and are passed to configure as CC / CXX.
14f95a42 1628 self.ccopts = ccopts
33dd32fd
SL
1629 # cflags contain non-ABI options like -g or -O and are passed to
1630 # configure as CFLAGS / CXXFLAGS.
1631 self.cflags = cflags
14f95a42
JM
1632
1633 def tool_name(self, tool):
1634 """Return the name of a cross-compilation tool."""
1635 ctool = '%s-%s' % (self.compiler.triplet, tool)
1636 if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1637 ctool = '%s %s' % (ctool, self.ccopts)
1638 return ctool
1639
1640 def build(self):
1641 """Generate commands to build this glibc."""
1642 builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1643 installdir = self.ctx.glibc_installdir(self.name)
1644 logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1645 self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1646 cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1647 cmdlist.add_command('check-compilers',
1648 ['test', '-f',
1649 os.path.join(self.compiler.installdir, 'ok')])
1650 cmdlist.use_path(self.compiler.bindir)
857c7d73 1651 self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
14f95a42
JM
1652 self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1653 logsdir)
1654
857c7d73 1655 def build_glibc(self, cmdlist, policy):
14f95a42
JM
1656 """Generate commands to build this glibc, either as part of a compiler
1657 build or with the bootstrapped compiler (and in the latter case, run
1658 tests as well)."""
857c7d73
FW
1659 cmdlist.create_use_dir(policy.builddir)
1660 policy.configure(cmdlist)
14f95a42
JM
1661 cmdlist.add_command('build', ['make'])
1662 cmdlist.add_command('install', ['make', 'install',
857c7d73 1663 'install_root=%s' % policy.installdir])
14f95a42
JM
1664 # GCC uses paths such as lib/../lib64, so make sure lib
1665 # directories always exist.
394b5bac 1666 mkdir_cmd = ['mkdir', '-p',
857c7d73
FW
1667 os.path.join(policy.installdir, 'lib')]
1668 if policy.use_usr:
1669 mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
394b5bac 1670 cmdlist.add_command('mkdir-lib', mkdir_cmd)
857c7d73 1671 policy.extra_commands(cmdlist)
14f95a42
JM
1672 cmdlist.cleanup_dir()
1673
07a44d23
FW
1674 def update_syscalls(self):
1675 if self.os == 'gnu':
1676 # Hurd does not have system call tables that need updating.
1677 return
1678
1679 policy = GlibcPolicyForUpdateSyscalls(self)
1680 logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1681 self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1682 cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1683 cmdlist.add_command('check-compilers',
1684 ['test', '-f',
1685 os.path.join(self.compiler.installdir, 'ok')])
1686 cmdlist.use_path(self.compiler.bindir)
1687
1688 install_linux_headers(policy.linux_policy, cmdlist)
1689
1690 cmdlist.create_use_dir(policy.builddir)
1691 policy.configure(cmdlist)
1692 cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1693 cmdlist.cleanup_dir()
1694 self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1695 cmdlist, logsdir)
14f95a42 1696
0c95f51d 1697class Command(object):
14f95a42
JM
1698 """A command run in the build process."""
1699
1700 def __init__(self, desc, num, dir, path, command, always_run=False):
1701 """Initialize a Command object."""
1702 self.dir = dir
1703 self.path = path
1704 self.desc = desc
1705 trans = str.maketrans({' ': '-'})
1706 self.logbase = '%03d-%s' % (num, desc.translate(trans))
1707 self.command = command
1708 self.always_run = always_run
1709
1710 @staticmethod
1711 def shell_make_quote_string(s):
1712 """Given a string not containing a newline, quote it for use by the
1713 shell and make."""
1714 assert '\n' not in s
1715 if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1716 return s
1717 strans = str.maketrans({"'": "'\\''"})
1718 s = "'%s'" % s.translate(strans)
1719 mtrans = str.maketrans({'$': '$$'})
1720 return s.translate(mtrans)
1721
1722 @staticmethod
1723 def shell_make_quote_list(l, translate_make):
1724 """Given a list of strings not containing newlines, quote them for use
1725 by the shell and make, returning a single string. If translate_make
1726 is true and the first string is 'make', change it to $(MAKE)."""
1727 l = [Command.shell_make_quote_string(s) for s in l]
1728 if translate_make and l[0] == 'make':
1729 l[0] = '$(MAKE)'
1730 return ' '.join(l)
1731
1732 def shell_make_quote(self):
1733 """Return this command quoted for the shell and make."""
1734 return self.shell_make_quote_list(self.command, True)
1735
1736
0c95f51d 1737class CommandList(object):
14f95a42
JM
1738 """A list of commands run in the build process."""
1739
1740 def __init__(self, desc, keep):
1741 """Initialize a CommandList object."""
1742 self.cmdlist = []
1743 self.dir = None
1744 self.path = None
1745 self.desc = [desc]
1746 self.keep = keep
1747
1748 def desc_txt(self, desc):
1749 """Return the description to use for a command."""
1750 return '%s %s' % (' '.join(self.desc), desc)
1751
1752 def use_dir(self, dir):
1753 """Set the default directory for subsequent commands."""
1754 self.dir = dir
1755
1756 def use_path(self, path):
1757 """Set a directory to be prepended to the PATH for subsequent
1758 commands."""
1759 self.path = path
1760
1761 def push_subdesc(self, subdesc):
1762 """Set the default subdescription for subsequent commands (e.g., the
1763 name of a component being built, within the series of commands
1764 building it)."""
1765 self.desc.append(subdesc)
1766
1767 def pop_subdesc(self):
1768 """Pop a subdescription from the list of descriptions."""
1769 self.desc.pop()
1770
1771 def create_use_dir(self, dir):
1772 """Remove and recreate a directory and use it for subsequent
1773 commands."""
1774 self.add_command_dir('rm', None, ['rm', '-rf', dir])
1775 self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1776 self.use_dir(dir)
1777
14f95a42
JM
1778 def add_command_dir(self, desc, dir, command, always_run=False):
1779 """Add a command to run in a given directory."""
1780 cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1781 command, always_run)
1782 self.cmdlist.append(cmd)
1783
1784 def add_command(self, desc, command, always_run=False):
1785 """Add a command to run in the default directory."""
1786 cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1787 self.path, command, always_run)
1788 self.cmdlist.append(cmd)
1789
1790 def cleanup_dir(self, desc='cleanup', dir=None):
1791 """Clean up a build directory. If no directory is specified, the
1792 default directory is cleaned up and ceases to be the default
1793 directory."""
1794 if dir is None:
1795 dir = self.dir
1796 self.use_dir(None)
1797 if self.keep != 'all':
1798 self.add_command_dir(desc, None, ['rm', '-rf', dir],
1799 always_run=(self.keep == 'none'))
1800
1801 def makefile_commands(self, wrapper, logsdir):
1802 """Return the sequence of commands in the form of text for a Makefile.
1803 The given wrapper script takes arguments: base of logs for
1804 previous command, or empty; base of logs for this command;
1805 description; directory; PATH addition; the command itself."""
1806 # prev_base is the base of the name for logs of the previous
1807 # command that is not always-run (that is, a build command,
1808 # whose failure should stop subsequent build commands from
1809 # being run, as opposed to a cleanup command, which is run
1810 # even if previous commands failed).
1811 prev_base = ''
1812 cmds = []
1813 for c in self.cmdlist:
1814 ctxt = c.shell_make_quote()
1815 if prev_base and not c.always_run:
1816 prev_log = os.path.join(logsdir, prev_base)
1817 else:
1818 prev_log = ''
1819 this_log = os.path.join(logsdir, c.logbase)
1820 if not c.always_run:
1821 prev_base = c.logbase
1822 if c.dir is None:
1823 dir = ''
1824 else:
1825 dir = c.dir
1826 if c.path is None:
1827 path = ''
1828 else:
1829 path = c.path
1830 prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1831 prelim_txt = Command.shell_make_quote_list(prelims, False)
1832 cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1833 return '\n'.join(cmds)
1834
bf469f0c
JM
1835 def status_logs(self, logsdir):
1836 """Return the list of log files with command status."""
1837 return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1838 for c in self.cmdlist]
1839
14f95a42
JM
1840
1841def get_parser():
1842 """Return an argument parser for this module."""
1843 parser = argparse.ArgumentParser(description=__doc__)
1844 parser.add_argument('-j', dest='parallelism',
1845 help='Run this number of jobs in parallel',
1846 type=int, default=os.cpu_count())
1847 parser.add_argument('--keep', dest='keep',
1848 help='Whether to keep all build directories, '
1849 'none or only those from failed builds',
1850 default='none', choices=('none', 'all', 'failed'))
02c78f02
JM
1851 parser.add_argument('--replace-sources', action='store_true',
1852 help='Remove and replace source directories '
1853 'with the wrong version of a component')
297635d8
JM
1854 parser.add_argument('--strip', action='store_true',
1855 help='Strip installed glibc libraries')
b1176270
JM
1856 parser.add_argument('--full-gcc', action='store_true',
1857 help='Build GCC with all languages and libsanitizer')
c592721a
FW
1858 parser.add_argument('--shallow', action='store_true',
1859 help='Do not download Git history during checkout')
14f95a42
JM
1860 parser.add_argument('topdir',
1861 help='Toplevel working directory')
1862 parser.add_argument('action',
1863 help='What to do',
a1f6a9ab 1864 choices=('checkout', 'bot-cycle', 'bot',
07a44d23 1865 'host-libraries', 'compilers', 'glibcs',
feaa1506
FW
1866 'update-syscalls', 'list-compilers',
1867 'list-glibcs'))
14f95a42
JM
1868 parser.add_argument('configs',
1869 help='Versions to check out or configurations to build',
1870 nargs='*')
1871 return parser
1872
1873
1874def main(argv):
1875 """The main entry point."""
1876 parser = get_parser()
1877 opts = parser.parse_args(argv)
1878 topdir = os.path.abspath(opts.topdir)
02c78f02 1879 ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
c592721a
FW
1880 opts.strip, opts.full_gcc, opts.action,
1881 shallow=opts.shallow)
14f95a42
JM
1882 ctx.run_builds(opts.action, opts.configs)
1883
1884
1885if __name__ == '__main__':
1886 main(sys.argv[1:])