2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <http://www.gnu.org/licenses/>.
20 """Build many configurations of glibc.
22 This script takes as arguments a directory name (containing a src
23 subdirectory with sources of the relevant toolchain components) and a
24 description of what to do: 'checkout', to check out sources into that
25 directory, 'bot-cycle', to run a series of checkout and build steps,
26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27 libraries required by the toolchain, 'compilers', to build
28 cross-compilers for various configurations, or 'glibcs', to build
29 glibc for various configurations and run the compilation parts of the
30 testsuite. Subsequent arguments name the versions of components to
31 check out (<component>-<version), for 'checkout', or, for actions
32 other than 'checkout' and 'bot-cycle', name configurations for which
33 compilers or glibc are to be built.
39 import email
.mime
.text
53 class Context(object):
54 """The global state associated with builds in a given directory."""
56 def __init__(self
, topdir
, parallelism
, keep
, replace_sources
, strip
,
58 """Initialize the context."""
60 self
.parallelism
= parallelism
62 self
.replace_sources
= replace_sources
64 self
.srcdir
= os
.path
.join(topdir
, 'src')
65 self
.versions_json
= os
.path
.join(self
.srcdir
, 'versions.json')
66 self
.build_state_json
= os
.path
.join(topdir
, 'build-state.json')
67 self
.bot_config_json
= os
.path
.join(topdir
, 'bot-config.json')
68 self
.installdir
= os
.path
.join(topdir
, 'install')
69 self
.host_libraries_installdir
= os
.path
.join(self
.installdir
,
71 self
.builddir
= os
.path
.join(topdir
, 'build')
72 self
.logsdir
= os
.path
.join(topdir
, 'logs')
73 self
.logsdir_old
= os
.path
.join(topdir
, 'logs-old')
74 self
.makefile
= os
.path
.join(self
.builddir
, 'Makefile')
75 self
.wrapper
= os
.path
.join(self
.builddir
, 'wrapper')
76 self
.save_logs
= os
.path
.join(self
.builddir
, 'save-logs')
77 self
.script_text
= self
.get_script_text()
78 if action
!= 'checkout':
79 self
.build_triplet
= self
.get_build_triplet()
80 self
.glibc_version
= self
.get_glibc_version()
82 self
.glibc_configs
= {}
83 self
.makefile_pieces
= ['.PHONY: all\n']
84 self
.add_all_configs()
85 self
.load_versions_json()
86 self
.load_build_state_json()
87 self
.status_log_list
= []
89 def get_script_text(self
):
90 """Return the text of this script."""
91 with
open(sys
.argv
[0], 'r') as f
:
95 """Re-execute this script with the same arguments."""
97 os
.execv(sys
.executable
, [sys
.executable
] + sys
.argv
)
99 def get_build_triplet(self
):
100 """Determine the build triplet with config.guess."""
101 config_guess
= os
.path
.join(self
.component_srcdir('gcc'),
103 cg_out
= subprocess
.run([config_guess
], stdout
=subprocess
.PIPE
,
104 check
=True, universal_newlines
=True).stdout
105 return cg_out
.rstrip()
107 def get_glibc_version(self
):
108 """Determine the glibc version number (major.minor)."""
109 version_h
= os
.path
.join(self
.component_srcdir('glibc'), 'version.h')
110 with
open(version_h
, 'r') as f
:
111 lines
= f
.readlines()
112 starttext
= '#define VERSION "'
114 if l
.startswith(starttext
):
115 l
= l
[len(starttext
):]
117 m
= re
.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l
)
118 return '%s.%s' % m
.group(1, 2)
119 print('error: could not determine glibc version')
122 def add_all_configs(self
):
123 """Add all known glibc build configurations."""
124 # On architectures missing __builtin_trap support, these
125 # options may be needed as a workaround; see
126 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
127 no_isolate
= ('-fno-isolate-erroneous-paths-dereference'
128 ' -fno-isolate-erroneous-paths-attribute')
129 self
.add_config(arch
='aarch64',
131 self
.add_config(arch
='aarch64_be',
133 self
.add_config(arch
='alpha',
135 self
.add_config(arch
='arm',
136 os_name
='linux-gnueabi')
137 self
.add_config(arch
='armeb',
138 os_name
='linux-gnueabi')
139 self
.add_config(arch
='armeb',
140 os_name
='linux-gnueabi',
142 gcc_cfg
=['--with-arch=armv7-a'])
143 self
.add_config(arch
='arm',
144 os_name
='linux-gnueabihf')
145 self
.add_config(arch
='armeb',
146 os_name
='linux-gnueabihf')
147 self
.add_config(arch
='armeb',
148 os_name
='linux-gnueabihf',
150 gcc_cfg
=['--with-arch=armv7-a'])
151 self
.add_config(arch
='hppa',
153 self
.add_config(arch
='ia64',
155 first_gcc_cfg
=['--with-system-libunwind'])
156 self
.add_config(arch
='m68k',
158 gcc_cfg
=['--disable-multilib'])
159 self
.add_config(arch
='m68k',
162 gcc_cfg
=['--with-arch=cf', '--disable-multilib'])
163 self
.add_config(arch
='microblaze',
165 gcc_cfg
=['--disable-multilib'])
166 self
.add_config(arch
='microblazeel',
168 gcc_cfg
=['--disable-multilib'])
169 self
.add_config(arch
='mips64',
171 gcc_cfg
=['--with-mips-plt'],
172 glibcs
=[{'variant': 'n32'},
174 'ccopts': '-mabi=32'},
176 'ccopts': '-mabi=64'}])
177 self
.add_config(arch
='mips64',
180 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
181 glibcs
=[{'variant': 'n32-soft',
182 'cfg': ['--without-fp']},
185 'ccopts': '-mabi=32',
186 'cfg': ['--without-fp']},
187 {'variant': 'n64-soft',
188 'ccopts': '-mabi=64',
189 'cfg': ['--without-fp']}])
190 self
.add_config(arch
='mips64',
193 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
194 '--with-arch-64=mips64r2',
195 '--with-arch-32=mips32r2'],
196 glibcs
=[{'variant': 'n32-nan2008'},
197 {'variant': 'nan2008',
199 'ccopts': '-mabi=32'},
200 {'variant': 'n64-nan2008',
201 'ccopts': '-mabi=64'}])
202 self
.add_config(arch
='mips64',
204 variant
='nan2008-soft',
205 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
206 '--with-arch-64=mips64r2',
207 '--with-arch-32=mips32r2',
208 '--with-float=soft'],
209 glibcs
=[{'variant': 'n32-nan2008-soft',
210 'cfg': ['--without-fp']},
211 {'variant': 'nan2008-soft',
213 'ccopts': '-mabi=32',
214 'cfg': ['--without-fp']},
215 {'variant': 'n64-nan2008-soft',
216 'ccopts': '-mabi=64',
217 'cfg': ['--without-fp']}])
218 self
.add_config(arch
='mips64el',
220 gcc_cfg
=['--with-mips-plt'],
221 glibcs
=[{'variant': 'n32'},
223 'ccopts': '-mabi=32'},
225 'ccopts': '-mabi=64'}])
226 self
.add_config(arch
='mips64el',
229 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
230 glibcs
=[{'variant': 'n32-soft',
231 'cfg': ['--without-fp']},
234 'ccopts': '-mabi=32',
235 'cfg': ['--without-fp']},
236 {'variant': 'n64-soft',
237 'ccopts': '-mabi=64',
238 'cfg': ['--without-fp']}])
239 self
.add_config(arch
='mips64el',
242 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
243 '--with-arch-64=mips64r2',
244 '--with-arch-32=mips32r2'],
245 glibcs
=[{'variant': 'n32-nan2008'},
246 {'variant': 'nan2008',
248 'ccopts': '-mabi=32'},
249 {'variant': 'n64-nan2008',
250 'ccopts': '-mabi=64'}])
251 self
.add_config(arch
='mips64el',
253 variant
='nan2008-soft',
254 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
255 '--with-arch-64=mips64r2',
256 '--with-arch-32=mips32r2',
257 '--with-float=soft'],
258 glibcs
=[{'variant': 'n32-nan2008-soft',
259 'cfg': ['--without-fp']},
260 {'variant': 'nan2008-soft',
262 'ccopts': '-mabi=32',
263 'cfg': ['--without-fp']},
264 {'variant': 'n64-nan2008-soft',
265 'ccopts': '-mabi=64',
266 'cfg': ['--without-fp']}])
267 self
.add_config(arch
='nios2',
269 self
.add_config(arch
='powerpc',
271 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
272 self
.add_config(arch
='powerpc',
275 gcc_cfg
=['--disable-multilib', '--with-float=soft',
276 '--enable-secureplt'],
277 glibcs
=[{'variant': 'soft', 'cfg': ['--without-fp']}])
278 self
.add_config(arch
='powerpc64',
280 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
281 self
.add_config(arch
='powerpc64le',
283 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
284 self
.add_config(arch
='powerpc',
285 os_name
='linux-gnuspe',
286 gcc_cfg
=['--disable-multilib', '--enable-secureplt',
287 '--enable-e500-double'],
288 glibcs
=[{'cfg': ['--without-fp']}])
289 self
.add_config(arch
='powerpc',
290 os_name
='linux-gnuspe',
292 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
293 glibcs
=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
294 self
.add_config(arch
='s390x',
297 {'arch': 's390', 'ccopts': '-m31'}])
298 self
.add_config(arch
='sh3',
300 glibcs
=[{'ccopts': no_isolate
}])
301 self
.add_config(arch
='sh3eb',
303 glibcs
=[{'ccopts': no_isolate
}])
304 self
.add_config(arch
='sh4',
306 glibcs
=[{'ccopts': no_isolate
}])
307 self
.add_config(arch
='sh4eb',
309 glibcs
=[{'ccopts': no_isolate
}])
310 self
.add_config(arch
='sh4',
313 gcc_cfg
=['--without-fp'],
314 glibcs
=[{'variant': 'soft',
315 'cfg': ['--without-fp'],
316 'ccopts': no_isolate
}])
317 self
.add_config(arch
='sh4eb',
320 gcc_cfg
=['--without-fp'],
321 glibcs
=[{'variant': 'soft',
322 'cfg': ['--without-fp'],
323 'ccopts': no_isolate
}])
324 self
.add_config(arch
='sparc64',
328 'ccopts': '-m32 -mlong-double-128'}])
329 self
.add_config(arch
='tilegx',
332 {'variant': '32', 'ccopts': '-m32'}])
333 self
.add_config(arch
='tilegxbe',
336 {'variant': '32', 'ccopts': '-m32'}])
337 self
.add_config(arch
='tilepro',
339 self
.add_config(arch
='x86_64',
341 gcc_cfg
=['--with-multilib-list=m64,m32,mx32'],
343 {'variant': 'x32', 'ccopts': '-mx32'},
344 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
345 extra_glibcs
=[{'variant': 'disable-multi-arch',
346 'cfg': ['--disable-multi-arch']},
347 {'variant': 'disable-multi-arch',
349 'ccopts': '-m32 -march=i686',
350 'cfg': ['--disable-multi-arch']},
352 'ccopts': '-m32 -march=i486'},
354 'ccopts': '-m32 -march=i586'}])
356 def add_config(self
, **args
):
357 """Add an individual build configuration."""
358 cfg
= Config(self
, **args
)
359 if cfg
.name
in self
.configs
:
360 print('error: duplicate config %s' % cfg
.name
)
362 self
.configs
[cfg
.name
] = cfg
363 for c
in cfg
.all_glibcs
:
364 if c
.name
in self
.glibc_configs
:
365 print('error: duplicate glibc config %s' % c
.name
)
367 self
.glibc_configs
[c
.name
] = c
369 def component_srcdir(self
, component
):
370 """Return the source directory for a given component, e.g. gcc."""
371 return os
.path
.join(self
.srcdir
, component
)
373 def component_builddir(self
, action
, config
, component
, subconfig
=None):
374 """Return the directory to use for a build."""
377 assert subconfig
is None
378 return os
.path
.join(self
.builddir
, action
, component
)
379 if subconfig
is None:
380 return os
.path
.join(self
.builddir
, action
, config
, component
)
382 # glibc build as part of compiler build.
383 return os
.path
.join(self
.builddir
, action
, config
, component
,
386 def compiler_installdir(self
, config
):
387 """Return the directory in which to install a compiler."""
388 return os
.path
.join(self
.installdir
, 'compilers', config
)
390 def compiler_bindir(self
, config
):
391 """Return the directory in which to find compiler binaries."""
392 return os
.path
.join(self
.compiler_installdir(config
), 'bin')
394 def compiler_sysroot(self
, config
):
395 """Return the sysroot directory for a compiler."""
396 return os
.path
.join(self
.compiler_installdir(config
), 'sysroot')
398 def glibc_installdir(self
, config
):
399 """Return the directory in which to install glibc."""
400 return os
.path
.join(self
.installdir
, 'glibcs', config
)
402 def run_builds(self
, action
, configs
):
403 """Run the requested builds."""
404 if action
== 'checkout':
405 self
.checkout(configs
)
407 if action
== 'bot-cycle':
409 print('error: configurations specified for bot-cycle')
415 print('error: configurations specified for bot')
419 if action
== 'host-libraries' and configs
:
420 print('error: configurations specified for host-libraries')
422 self
.clear_last_build_state(action
)
423 build_time
= datetime
.datetime
.utcnow()
424 if action
== 'host-libraries':
425 build_components
= ('gmp', 'mpfr', 'mpc')
428 self
.build_host_libraries()
429 elif action
== 'compilers':
430 build_components
= ('binutils', 'gcc', 'glibc', 'linux')
431 old_components
= ('gmp', 'mpfr', 'mpc')
432 old_versions
= self
.build_state
['host-libraries']['build-versions']
433 self
.build_compilers(configs
)
435 build_components
= ('glibc',)
436 old_components
= ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
437 old_versions
= self
.build_state
['compilers']['build-versions']
438 self
.build_glibcs(configs
)
442 # Partial build, do not update stored state.
445 for k
in build_components
:
446 if k
in self
.versions
:
447 build_versions
[k
] = {'version': self
.versions
[k
]['version'],
448 'revision': self
.versions
[k
]['revision']}
449 for k
in old_components
:
450 if k
in old_versions
:
451 build_versions
[k
] = {'version': old_versions
[k
]['version'],
452 'revision': old_versions
[k
]['revision']}
453 self
.update_build_state(action
, build_time
, build_versions
)
456 def remove_dirs(*args
):
457 """Remove directories and their contents if they exist."""
459 shutil
.rmtree(dir, ignore_errors
=True)
462 def remove_recreate_dirs(*args
):
463 """Remove directories if they exist, and create them as empty."""
464 Context
.remove_dirs(*args
)
466 os
.makedirs(dir, exist_ok
=True)
468 def add_makefile_cmdlist(self
, target
, cmdlist
, logsdir
):
469 """Add makefile text for a list of commands."""
470 commands
= cmdlist
.makefile_commands(self
.wrapper
, logsdir
)
471 self
.makefile_pieces
.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
472 (target
, target
, target
, commands
))
473 self
.status_log_list
.extend(cmdlist
.status_logs(logsdir
))
475 def write_files(self
):
476 """Write out the Makefile and wrapper script."""
477 mftext
= ''.join(self
.makefile_pieces
)
478 with
open(self
.makefile
, 'w') as f
:
488 'prev_status=$prev_base-status.txt\n'
489 'this_status=$this_base-status.txt\n'
490 'this_log=$this_base-log.txt\n'
491 'date > "$this_log"\n'
492 'echo >> "$this_log"\n'
493 'echo "Description: $desc" >> "$this_log"\n'
494 'printf "%s" "Command:" >> "$this_log"\n'
495 'for word in "$@"; do\n'
496 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
497 ' printf " %s" "$word"\n'
500 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
503 'done >> "$this_log"\n'
504 'echo >> "$this_log"\n'
505 'echo "Directory: $dir" >> "$this_log"\n'
506 'echo "Path addition: $path" >> "$this_log"\n'
507 'echo >> "$this_log"\n'
510 ' echo >> "$this_log"\n'
511 ' echo "$1: $desc" > "$this_status"\n'
512 ' echo "$1: $desc" >> "$this_log"\n'
513 ' echo >> "$this_log"\n'
514 ' date >> "$this_log"\n'
515 ' echo "$1: $desc"\n'
520 ' if [ "$1" != "0" ]; then\n'
521 ' record_status FAIL\n'
524 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
525 ' record_status UNRESOLVED\n'
527 'if [ "$dir" ]; then\n'
529 ' check_error "$?"\n'
531 'if [ "$path" ]; then\n'
532 ' PATH=$path:$PATH\n'
534 '"$@" < /dev/null >> "$this_log" 2>&1\n'
536 'record_status PASS\n')
537 with
open(self
.wrapper
, 'w') as f
:
538 f
.write(wrapper_text
)
540 mode_exec
= (stat
.S_IRWXU|stat
.S_IRGRP|stat
.S_IXGRP|
541 stat
.S_IROTH|stat
.S_IXOTH
)
542 os
.chmod(self
.wrapper
, mode_exec
)
545 'if ! [ -f tests.sum ]; then\n'
546 ' echo "No test summary available."\n'
551 ' echo "Contents of $1:"\n'
555 ' echo "End of contents of $1."\n'
558 'save_file tests.sum\n'
559 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
560 'for t in $non_pass_tests; do\n'
561 ' if [ -f "$t.out" ]; then\n'
562 ' save_file "$t.out"\n'
565 with
open(self
.save_logs
, 'w') as f
:
566 f
.write(save_logs_text
)
567 os
.chmod(self
.save_logs
, mode_exec
)
570 """Do the actual build."""
571 cmd
= ['make', '-j%d' % self
.parallelism
]
572 subprocess
.run(cmd
, cwd
=self
.builddir
, check
=True)
574 def build_host_libraries(self
):
575 """Build the host libraries."""
576 installdir
= self
.host_libraries_installdir
577 builddir
= os
.path
.join(self
.builddir
, 'host-libraries')
578 logsdir
= os
.path
.join(self
.logsdir
, 'host-libraries')
579 self
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
580 cmdlist
= CommandList('host-libraries', self
.keep
)
581 self
.build_host_library(cmdlist
, 'gmp')
582 self
.build_host_library(cmdlist
, 'mpfr',
583 ['--with-gmp=%s' % installdir
])
584 self
.build_host_library(cmdlist
, 'mpc',
585 ['--with-gmp=%s' % installdir
,
586 '--with-mpfr=%s' % installdir
])
587 cmdlist
.add_command('done', ['touch', os
.path
.join(installdir
, 'ok')])
588 self
.add_makefile_cmdlist('host-libraries', cmdlist
, logsdir
)
590 def build_host_library(self
, cmdlist
, lib
, extra_opts
=None):
591 """Build one host library."""
592 srcdir
= self
.component_srcdir(lib
)
593 builddir
= self
.component_builddir('host-libraries', None, lib
)
594 installdir
= self
.host_libraries_installdir
595 cmdlist
.push_subdesc(lib
)
596 cmdlist
.create_use_dir(builddir
)
597 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
598 '--prefix=%s' % installdir
,
601 cfg_cmd
.extend (extra_opts
)
602 cmdlist
.add_command('configure', cfg_cmd
)
603 cmdlist
.add_command('build', ['make'])
604 cmdlist
.add_command('check', ['make', 'check'])
605 cmdlist
.add_command('install', ['make', 'install'])
606 cmdlist
.cleanup_dir()
607 cmdlist
.pop_subdesc()
609 def build_compilers(self
, configs
):
610 """Build the compilers."""
612 self
.remove_dirs(os
.path
.join(self
.builddir
, 'compilers'))
613 self
.remove_dirs(os
.path
.join(self
.installdir
, 'compilers'))
614 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'compilers'))
615 configs
= sorted(self
.configs
.keys())
617 self
.configs
[c
].build()
619 def build_glibcs(self
, configs
):
620 """Build the glibcs."""
622 self
.remove_dirs(os
.path
.join(self
.builddir
, 'glibcs'))
623 self
.remove_dirs(os
.path
.join(self
.installdir
, 'glibcs'))
624 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'glibcs'))
625 configs
= sorted(self
.glibc_configs
.keys())
627 self
.glibc_configs
[c
].build()
629 def load_versions_json(self
):
630 """Load information about source directory versions."""
631 if not os
.access(self
.versions_json
, os
.F_OK
):
634 with
open(self
.versions_json
, 'r') as f
:
635 self
.versions
= json
.load(f
)
637 def store_json(self
, data
, filename
):
638 """Store information in a JSON file."""
639 filename_tmp
= filename
+ '.tmp'
640 with
open(filename_tmp
, 'w') as f
:
641 json
.dump(data
, f
, indent
=2, sort_keys
=True)
642 os
.rename(filename_tmp
, filename
)
644 def store_versions_json(self
):
645 """Store information about source directory versions."""
646 self
.store_json(self
.versions
, self
.versions_json
)
648 def set_component_version(self
, component
, version
, explicit
, revision
):
649 """Set the version information for a component."""
650 self
.versions
[component
] = {'version': version
,
651 'explicit': explicit
,
652 'revision': revision
}
653 self
.store_versions_json()
655 def checkout(self
, versions
):
656 """Check out the desired component versions."""
657 default_versions
= {'binutils': 'vcs-2.28',
659 'glibc': 'vcs-mainline',
665 explicit_versions
= {}
668 for k
in default_versions
.keys():
672 if k
in use_versions
:
673 print('error: multiple versions for %s' % k
)
676 explicit_versions
[k
] = True
680 print('error: unknown component in %s' % v
)
682 for k
in default_versions
.keys():
683 if k
not in use_versions
:
684 if k
in self
.versions
and self
.versions
[k
]['explicit']:
685 use_versions
[k
] = self
.versions
[k
]['version']
686 explicit_versions
[k
] = True
688 use_versions
[k
] = default_versions
[k
]
689 explicit_versions
[k
] = False
690 os
.makedirs(self
.srcdir
, exist_ok
=True)
691 for k
in sorted(default_versions
.keys()):
692 update
= os
.access(self
.component_srcdir(k
), os
.F_OK
)
695 k
in self
.versions
and
696 v
!= self
.versions
[k
]['version']):
697 if not self
.replace_sources
:
698 print('error: version of %s has changed from %s to %s, '
699 'use --replace-sources to check out again' %
700 (k
, self
.versions
[k
]['version'], v
))
702 shutil
.rmtree(self
.component_srcdir(k
))
704 if v
.startswith('vcs-'):
705 revision
= self
.checkout_vcs(k
, v
[4:], update
)
707 self
.checkout_tar(k
, v
, update
)
709 self
.set_component_version(k
, v
, explicit_versions
[k
], revision
)
710 if self
.get_script_text() != self
.script_text
:
711 # Rerun the checkout process in case the updated script
712 # uses different default versions or new components.
715 def checkout_vcs(self
, component
, version
, update
):
716 """Check out the given version of the given component from version
717 control. Return a revision identifier."""
718 if component
== 'binutils':
719 git_url
= 'git://sourceware.org/git/binutils-gdb.git'
720 if version
== 'mainline':
721 git_branch
= 'master'
723 trans
= str.maketrans({'.': '_'})
724 git_branch
= 'binutils-%s-branch' % version
.translate(trans
)
725 return self
.git_checkout(component
, git_url
, git_branch
, update
)
726 elif component
== 'gcc':
727 if version
== 'mainline':
730 trans
= str.maketrans({'.': '_'})
731 branch
= 'branches/gcc-%s-branch' % version
.translate(trans
)
732 svn_url
= 'svn://gcc.gnu.org/svn/gcc/%s' % branch
733 return self
.gcc_checkout(svn_url
, update
)
734 elif component
== 'glibc':
735 git_url
= 'git://sourceware.org/git/glibc.git'
736 if version
== 'mainline':
737 git_branch
= 'master'
739 git_branch
= 'release/%s/master' % version
740 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
741 self
.fix_glibc_timestamps()
744 print('error: component %s coming from VCS' % component
)
747 def git_checkout(self
, component
, git_url
, git_branch
, update
):
748 """Check out a component from git. Return a commit identifier."""
750 subprocess
.run(['git', 'remote', 'prune', 'origin'],
751 cwd
=self
.component_srcdir(component
), check
=True)
752 subprocess
.run(['git', 'pull', '-q'],
753 cwd
=self
.component_srcdir(component
), check
=True)
755 subprocess
.run(['git', 'clone', '-q', '-b', git_branch
, git_url
,
756 self
.component_srcdir(component
)], check
=True)
757 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
758 cwd
=self
.component_srcdir(component
),
759 stdout
=subprocess
.PIPE
,
760 check
=True, universal_newlines
=True).stdout
763 def fix_glibc_timestamps(self
):
764 """Fix timestamps in a glibc checkout."""
765 # Ensure that builds do not try to regenerate generated files
766 # in the source tree.
767 srcdir
= self
.component_srcdir('glibc')
768 for dirpath
, dirnames
, filenames
in os
.walk(srcdir
):
770 if (f
== 'configure' or
771 f
== 'preconfigure' or
772 f
.endswith('-kw.h')):
773 to_touch
= os
.path
.join(dirpath
, f
)
774 subprocess
.run(['touch', to_touch
], check
=True)
776 def gcc_checkout(self
, svn_url
, update
):
777 """Check out GCC from SVN. Return the revision number."""
779 subprocess
.run(['svn', 'co', '-q', svn_url
,
780 self
.component_srcdir('gcc')], check
=True)
781 subprocess
.run(['contrib/gcc_update', '--silent'],
782 cwd
=self
.component_srcdir('gcc'), check
=True)
783 r
= subprocess
.run(['svnversion', self
.component_srcdir('gcc')],
784 stdout
=subprocess
.PIPE
,
785 check
=True, universal_newlines
=True).stdout
788 def checkout_tar(self
, component
, version
, update
):
789 """Check out the given version of the given component from a
793 url_map
= {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
794 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
795 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
796 'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
797 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
798 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
799 if component
not in url_map
:
800 print('error: component %s coming from tarball' % component
)
802 url
= url_map
[component
] % {'version': version
}
803 filename
= os
.path
.join(self
.srcdir
, url
.split('/')[-1])
804 response
= urllib
.request
.urlopen(url
)
805 data
= response
.read()
806 with
open(filename
, 'wb') as f
:
808 subprocess
.run(['tar', '-C', self
.srcdir
, '-x', '-f', filename
],
810 os
.rename(os
.path
.join(self
.srcdir
, '%s-%s' % (component
, version
)),
811 self
.component_srcdir(component
))
814 def load_build_state_json(self
):
815 """Load information about the state of previous builds."""
816 if os
.access(self
.build_state_json
, os
.F_OK
):
817 with
open(self
.build_state_json
, 'r') as f
:
818 self
.build_state
= json
.load(f
)
820 self
.build_state
= {}
821 for k
in ('host-libraries', 'compilers', 'glibcs'):
822 if k
not in self
.build_state
:
823 self
.build_state
[k
] = {}
824 if 'build-time' not in self
.build_state
[k
]:
825 self
.build_state
[k
]['build-time'] = ''
826 if 'build-versions' not in self
.build_state
[k
]:
827 self
.build_state
[k
]['build-versions'] = {}
828 if 'build-results' not in self
.build_state
[k
]:
829 self
.build_state
[k
]['build-results'] = {}
830 if 'result-changes' not in self
.build_state
[k
]:
831 self
.build_state
[k
]['result-changes'] = {}
832 if 'ever-passed' not in self
.build_state
[k
]:
833 self
.build_state
[k
]['ever-passed'] = []
835 def store_build_state_json(self
):
836 """Store information about the state of previous builds."""
837 self
.store_json(self
.build_state
, self
.build_state_json
)
839 def clear_last_build_state(self
, action
):
840 """Clear information about the state of part of the build."""
841 # We clear the last build time and versions when starting a
842 # new build. The results of the last build are kept around,
843 # as comparison is still meaningful if this build is aborted
844 # and a new one started.
845 self
.build_state
[action
]['build-time'] = ''
846 self
.build_state
[action
]['build-versions'] = {}
847 self
.store_build_state_json()
849 def update_build_state(self
, action
, build_time
, build_versions
):
850 """Update the build state after a build."""
851 build_time
= build_time
.replace(microsecond
=0)
852 self
.build_state
[action
]['build-time'] = str(build_time
)
853 self
.build_state
[action
]['build-versions'] = build_versions
855 for log
in self
.status_log_list
:
856 with
open(log
, 'r') as f
:
858 log_text
= log_text
.rstrip()
859 m
= re
.fullmatch('([A-Z]+): (.*)', log_text
)
861 test_name
= m
.group(2)
862 assert test_name
not in build_results
863 build_results
[test_name
] = result
864 old_build_results
= self
.build_state
[action
]['build-results']
865 self
.build_state
[action
]['build-results'] = build_results
867 all_tests
= set(old_build_results
.keys()) |
set(build_results
.keys())
869 if t
in old_build_results
:
870 old_res
= old_build_results
[t
]
872 old_res
= '(New test)'
873 if t
in build_results
:
874 new_res
= build_results
[t
]
876 new_res
= '(Test removed)'
877 if old_res
!= new_res
:
878 result_changes
[t
] = '%s -> %s' % (old_res
, new_res
)
879 self
.build_state
[action
]['result-changes'] = result_changes
880 old_ever_passed
= {t
for t
in self
.build_state
[action
]['ever-passed']
881 if t
in build_results
}
882 new_passes
= {t
for t
in build_results
if build_results
[t
] == 'PASS'}
883 self
.build_state
[action
]['ever-passed'] = sorted(old_ever_passed |
885 self
.store_build_state_json()
887 def load_bot_config_json(self
):
888 """Load bot configuration."""
889 with
open(self
.bot_config_json
, 'r') as f
:
890 self
.bot_config
= json
.load(f
)
892 def part_build_old(self
, action
, delay
):
893 """Return whether the last build for a given action was at least a
894 given number of seconds ago, or does not have a time recorded."""
895 old_time_str
= self
.build_state
[action
]['build-time']
898 old_time
= datetime
.datetime
.strptime(old_time_str
,
900 new_time
= datetime
.datetime
.utcnow()
901 delta
= new_time
- old_time
902 return delta
.total_seconds() >= delay
905 """Run a single round of checkout and builds."""
906 print('Bot cycle starting %s.' % str(datetime
.datetime
.utcnow()))
907 self
.load_bot_config_json()
908 actions
= ('host-libraries', 'compilers', 'glibcs')
909 self
.bot_run_self(['--replace-sources'], 'checkout')
910 self
.load_versions_json()
911 if self
.get_script_text() != self
.script_text
:
912 print('Script changed, re-execing.')
913 # On script change, all parts of the build should be rerun.
915 self
.clear_last_build_state(a
)
917 check_components
= {'host-libraries': ('gmp', 'mpfr', 'mpc'),
918 'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
919 'glibcs': ('glibc',)}
922 build_vers
= self
.build_state
[a
]['build-versions']
923 must_build
[a
] = False
924 if not self
.build_state
[a
]['build-time']:
928 for c
in check_components
[a
]:
930 old_vers
[c
] = build_vers
[c
]
931 new_vers
[c
] = {'version': self
.versions
[c
]['version'],
932 'revision': self
.versions
[c
]['revision']}
933 if new_vers
== old_vers
:
934 print('Versions for %s unchanged.' % a
)
936 print('Versions changed or rebuild forced for %s.' % a
)
937 if a
== 'compilers' and not self
.part_build_old(
938 a
, self
.bot_config
['compilers-rebuild-delay']):
939 print('Not requiring rebuild of compilers this soon.')
942 if must_build
['host-libraries']:
943 must_build
['compilers'] = True
944 if must_build
['compilers']:
945 must_build
['glibcs'] = True
948 print('Must rebuild %s.' % a
)
949 self
.clear_last_build_state(a
)
951 print('No need to rebuild %s.' % a
)
952 if os
.access(self
.logsdir
, os
.F_OK
):
953 shutil
.rmtree(self
.logsdir_old
, ignore_errors
=True)
954 shutil
.copytree(self
.logsdir
, self
.logsdir_old
)
957 build_time
= datetime
.datetime
.utcnow()
958 print('Rebuilding %s at %s.' % (a
, str(build_time
)))
959 self
.bot_run_self([], a
)
960 self
.load_build_state_json()
961 self
.bot_build_mail(a
, build_time
)
962 print('Bot cycle done at %s.' % str(datetime
.datetime
.utcnow()))
964 def bot_build_mail(self
, action
, build_time
):
965 """Send email with the results of a build."""
966 build_time
= build_time
.replace(microsecond
=0)
967 subject
= (self
.bot_config
['email-subject'] %
969 'build-time': str(build_time
)})
970 results
= self
.build_state
[action
]['build-results']
971 changes
= self
.build_state
[action
]['result-changes']
972 ever_passed
= set(self
.build_state
[action
]['ever-passed'])
973 versions
= self
.build_state
[action
]['build-versions']
974 new_regressions
= {k
for k
in changes
if changes
[k
] == 'PASS -> FAIL'}
975 all_regressions
= {k
for k
in ever_passed
if results
[k
] == 'FAIL'}
976 all_fails
= {k
for k
in results
if results
[k
] == 'FAIL'}
978 new_reg_list
= sorted(['FAIL: %s' % k
for k
in new_regressions
])
979 new_reg_text
= ('New regressions:\n\n%s\n\n' %
980 '\n'.join(new_reg_list
))
984 all_reg_list
= sorted(['FAIL: %s' % k
for k
in all_regressions
])
985 all_reg_text
= ('All regressions:\n\n%s\n\n' %
986 '\n'.join(all_reg_list
))
990 all_fail_list
= sorted(['FAIL: %s' % k
for k
in all_fails
])
991 all_fail_text
= ('All failures:\n\n%s\n\n' %
992 '\n'.join(all_fail_list
))
996 changes_list
= sorted(changes
.keys())
997 changes_list
= ['%s: %s' % (changes
[k
], k
) for k
in changes_list
]
998 changes_text
= ('All changed results:\n\n%s\n\n' %
999 '\n'.join(changes_list
))
1002 results_text
= (new_reg_text
+ all_reg_text
+ all_fail_text
+
1004 if not results_text
:
1005 results_text
= 'Clean build with unchanged results.\n\n'
1006 versions_list
= sorted(versions
.keys())
1007 versions_list
= ['%s: %s (%s)' % (k
, versions
[k
]['version'],
1008 versions
[k
]['revision'])
1009 for k
in versions_list
]
1010 versions_text
= ('Component versions for this build:\n\n%s\n' %
1011 '\n'.join(versions_list
))
1012 body_text
= results_text
+ versions_text
1013 msg
= email
.mime
.text
.MIMEText(body_text
)
1014 msg
['Subject'] = subject
1015 msg
['From'] = self
.bot_config
['email-from']
1016 msg
['To'] = self
.bot_config
['email-to']
1017 msg
['Message-ID'] = email
.utils
.make_msgid()
1018 msg
['Date'] = email
.utils
.format_datetime(datetime
.datetime
.utcnow())
1019 with smtplib
.SMTP(self
.bot_config
['email-server']) as s
:
1022 def bot_run_self(self
, opts
, action
, check
=True):
1023 """Run a copy of this script with given options."""
1024 cmd
= [sys
.executable
, sys
.argv
[0], '--keep=none',
1025 '-j%d' % self
.parallelism
]
1027 cmd
.extend([self
.topdir
, action
])
1029 subprocess
.run(cmd
, check
=check
)
1032 """Run repeated rounds of checkout and builds."""
1034 self
.load_bot_config_json()
1035 if not self
.bot_config
['run']:
1036 print('Bot exiting by request.')
1038 self
.bot_run_self([], 'bot-cycle', check
=False)
1039 self
.load_bot_config_json()
1040 if not self
.bot_config
['run']:
1041 print('Bot exiting by request.')
1043 time
.sleep(self
.bot_config
['delay'])
1044 if self
.get_script_text() != self
.script_text
:
1045 print('Script changed, bot re-execing.')
1049 class Config(object):
1050 """A configuration for building a compiler and associated libraries."""
1052 def __init__(self
, ctx
, arch
, os_name
, variant
=None, gcc_cfg
=None,
1053 first_gcc_cfg
=None, glibcs
=None, extra_glibcs
=None):
1054 """Initialize a Config object."""
1058 self
.variant
= variant
1060 self
.name
= '%s-%s' % (arch
, os_name
)
1062 self
.name
= '%s-%s-%s' % (arch
, os_name
, variant
)
1063 self
.triplet
= '%s-glibc-%s' % (arch
, os_name
)
1067 self
.gcc_cfg
= gcc_cfg
1068 if first_gcc_cfg
is None:
1069 self
.first_gcc_cfg
= []
1071 self
.first_gcc_cfg
= first_gcc_cfg
1073 glibcs
= [{'variant': variant
}]
1074 if extra_glibcs
is None:
1076 glibcs
= [Glibc(self
, **g
) for g
in glibcs
]
1077 extra_glibcs
= [Glibc(self
, **g
) for g
in extra_glibcs
]
1078 self
.all_glibcs
= glibcs
+ extra_glibcs
1079 self
.compiler_glibcs
= glibcs
1080 self
.installdir
= ctx
.compiler_installdir(self
.name
)
1081 self
.bindir
= ctx
.compiler_bindir(self
.name
)
1082 self
.sysroot
= ctx
.compiler_sysroot(self
.name
)
1083 self
.builddir
= os
.path
.join(ctx
.builddir
, 'compilers', self
.name
)
1084 self
.logsdir
= os
.path
.join(ctx
.logsdir
, 'compilers', self
.name
)
1086 def component_builddir(self
, component
):
1087 """Return the directory to use for a (non-glibc) build."""
1088 return self
.ctx
.component_builddir('compilers', self
.name
, component
)
1091 """Generate commands to build this compiler."""
1092 self
.ctx
.remove_recreate_dirs(self
.installdir
, self
.builddir
,
1094 cmdlist
= CommandList('compilers-%s' % self
.name
, self
.ctx
.keep
)
1095 cmdlist
.add_command('check-host-libraries',
1097 os
.path
.join(self
.ctx
.host_libraries_installdir
,
1099 cmdlist
.use_path(self
.bindir
)
1100 self
.build_cross_tool(cmdlist
, 'binutils', 'binutils',
1102 '--disable-libdecnumber',
1103 '--disable-readline',
1105 if self
.os
.startswith('linux'):
1106 self
.install_linux_headers(cmdlist
)
1107 self
.build_gcc(cmdlist
, True)
1108 for g
in self
.compiler_glibcs
:
1109 cmdlist
.push_subdesc('glibc')
1110 cmdlist
.push_subdesc(g
.name
)
1111 g
.build_glibc(cmdlist
, True)
1112 cmdlist
.pop_subdesc()
1113 cmdlist
.pop_subdesc()
1114 self
.build_gcc(cmdlist
, False)
1115 cmdlist
.add_command('done', ['touch',
1116 os
.path
.join(self
.installdir
, 'ok')])
1117 self
.ctx
.add_makefile_cmdlist('compilers-%s' % self
.name
, cmdlist
,
1120 def build_cross_tool(self
, cmdlist
, tool_src
, tool_build
, extra_opts
=None):
1121 """Build one cross tool."""
1122 srcdir
= self
.ctx
.component_srcdir(tool_src
)
1123 builddir
= self
.component_builddir(tool_build
)
1124 cmdlist
.push_subdesc(tool_build
)
1125 cmdlist
.create_use_dir(builddir
)
1126 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
1127 '--prefix=%s' % self
.installdir
,
1128 '--build=%s' % self
.ctx
.build_triplet
,
1129 '--host=%s' % self
.ctx
.build_triplet
,
1130 '--target=%s' % self
.triplet
,
1131 '--with-sysroot=%s' % self
.sysroot
]
1133 cfg_cmd
.extend(extra_opts
)
1134 cmdlist
.add_command('configure', cfg_cmd
)
1135 cmdlist
.add_command('build', ['make'])
1136 cmdlist
.add_command('install', ['make', 'install'])
1137 cmdlist
.cleanup_dir()
1138 cmdlist
.pop_subdesc()
1140 def install_linux_headers(self
, cmdlist
):
1141 """Install Linux kernel headers."""
1142 arch_map
= {'aarch64': 'arm64',
1152 'microblaze': 'microblaze',
1155 'powerpc': 'powerpc',
1163 if self
.arch
.startswith(k
):
1164 linux_arch
= arch_map
[k
]
1166 assert linux_arch
is not None
1167 srcdir
= self
.ctx
.component_srcdir('linux')
1168 builddir
= self
.component_builddir('linux')
1169 headers_dir
= os
.path
.join(self
.sysroot
, 'usr')
1170 cmdlist
.push_subdesc('linux')
1171 cmdlist
.create_use_dir(builddir
)
1172 cmdlist
.add_command('install-headers',
1173 ['make', '-C', srcdir
, 'O=%s' % builddir
,
1174 'ARCH=%s' % linux_arch
,
1175 'INSTALL_HDR_PATH=%s' % headers_dir
,
1177 cmdlist
.cleanup_dir()
1178 cmdlist
.pop_subdesc()
1180 def build_gcc(self
, cmdlist
, bootstrap
):
1182 # libsanitizer commonly breaks because of glibc header
1183 # changes, or on unusual targets. libssp is of little
1184 # relevance with glibc's own stack checking support.
1185 cfg_opts
= list(self
.gcc_cfg
)
1186 cfg_opts
+= ['--disable-libsanitizer', '--disable-libssp']
1187 host_libs
= self
.ctx
.host_libraries_installdir
1188 cfg_opts
+= ['--with-gmp=%s' % host_libs
,
1189 '--with-mpfr=%s' % host_libs
,
1190 '--with-mpc=%s' % host_libs
]
1192 tool_build
= 'gcc-first'
1193 # Building a static-only, C-only compiler that is
1194 # sufficient to build glibc. Various libraries and
1195 # features that may require libc headers must be disabled.
1196 # When configuring with a sysroot, --with-newlib is
1197 # required to define inhibit_libc (to stop some parts of
1198 # libgcc including libc headers); --without-headers is not
1200 cfg_opts
+= ['--enable-languages=c', '--disable-shared',
1201 '--disable-threads',
1202 '--disable-libatomic',
1203 '--disable-decimal-float',
1205 '--disable-libgomp',
1208 '--disable-libquadmath',
1209 '--without-headers', '--with-newlib',
1210 '--with-glibc-version=%s' % self
.ctx
.glibc_version
1212 cfg_opts
+= self
.first_gcc_cfg
1215 cfg_opts
+= ['--enable-languages=c,c++', '--enable-shared',
1217 self
.build_cross_tool(cmdlist
, 'gcc', tool_build
, cfg_opts
)
1220 class Glibc(object):
1221 """A configuration for building glibc."""
1223 def __init__(self
, compiler
, arch
=None, os_name
=None, variant
=None,
1224 cfg
=None, ccopts
=None):
1225 """Initialize a Glibc object."""
1226 self
.ctx
= compiler
.ctx
1227 self
.compiler
= compiler
1229 self
.arch
= compiler
.arch
1233 self
.os
= compiler
.os
1236 self
.variant
= variant
1238 self
.name
= '%s-%s' % (self
.arch
, self
.os
)
1240 self
.name
= '%s-%s-%s' % (self
.arch
, self
.os
, variant
)
1241 self
.triplet
= '%s-glibc-%s' % (self
.arch
, self
.os
)
1246 self
.ccopts
= ccopts
1248 def tool_name(self
, tool
):
1249 """Return the name of a cross-compilation tool."""
1250 ctool
= '%s-%s' % (self
.compiler
.triplet
, tool
)
1251 if self
.ccopts
and (tool
== 'gcc' or tool
== 'g++'):
1252 ctool
= '%s %s' % (ctool
, self
.ccopts
)
1256 """Generate commands to build this glibc."""
1257 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
, 'glibc')
1258 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1259 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'glibcs', self
.name
)
1260 self
.ctx
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
1261 cmdlist
= CommandList('glibcs-%s' % self
.name
, self
.ctx
.keep
)
1262 cmdlist
.add_command('check-compilers',
1264 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1265 cmdlist
.use_path(self
.compiler
.bindir
)
1266 self
.build_glibc(cmdlist
, False)
1267 self
.ctx
.add_makefile_cmdlist('glibcs-%s' % self
.name
, cmdlist
,
1270 def build_glibc(self
, cmdlist
, for_compiler
):
1271 """Generate commands to build this glibc, either as part of a compiler
1272 build or with the bootstrapped compiler (and in the latter case, run
1274 srcdir
= self
.ctx
.component_srcdir('glibc')
1276 builddir
= self
.ctx
.component_builddir('compilers',
1277 self
.compiler
.name
, 'glibc',
1279 installdir
= self
.compiler
.sysroot
1280 srcdir_copy
= self
.ctx
.component_builddir('compilers',
1285 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
,
1287 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1288 srcdir_copy
= self
.ctx
.component_builddir('glibcs', self
.name
,
1290 cmdlist
.create_use_dir(builddir
)
1291 # glibc builds write into the source directory, and even if
1292 # not intentionally there is a risk of bugs that involve
1293 # writing into the working directory. To avoid possible
1294 # concurrency issues, copy the source directory.
1295 cmdlist
.create_copy_dir(srcdir
, srcdir_copy
)
1296 cfg_cmd
= [os
.path
.join(srcdir_copy
, 'configure'),
1299 '--build=%s' % self
.ctx
.build_triplet
,
1300 '--host=%s' % self
.triplet
,
1301 'CC=%s' % self
.tool_name('gcc'),
1302 'CXX=%s' % self
.tool_name('g++'),
1303 'AR=%s' % self
.tool_name('ar'),
1304 'AS=%s' % self
.tool_name('as'),
1305 'LD=%s' % self
.tool_name('ld'),
1306 'NM=%s' % self
.tool_name('nm'),
1307 'OBJCOPY=%s' % self
.tool_name('objcopy'),
1308 'OBJDUMP=%s' % self
.tool_name('objdump'),
1309 'RANLIB=%s' % self
.tool_name('ranlib'),
1310 'READELF=%s' % self
.tool_name('readelf'),
1311 'STRIP=%s' % self
.tool_name('strip')]
1313 cmdlist
.add_command('configure', cfg_cmd
)
1314 cmdlist
.add_command('build', ['make'])
1315 cmdlist
.add_command('install', ['make', 'install',
1316 'install_root=%s' % installdir
])
1317 # GCC uses paths such as lib/../lib64, so make sure lib
1318 # directories always exist.
1319 cmdlist
.add_command('mkdir-lib', ['mkdir', '-p',
1320 os
.path
.join(installdir
, 'lib'),
1321 os
.path
.join(installdir
,
1323 if not for_compiler
:
1325 cmdlist
.add_command('strip',
1327 ('%s %s/lib*/*.so' %
1328 (self
.tool_name('strip'), installdir
))])
1329 cmdlist
.add_command('check', ['make', 'check'])
1330 cmdlist
.add_command('save-logs', [self
.ctx
.save_logs
],
1332 cmdlist
.cleanup_dir('cleanup-src', srcdir_copy
)
1333 cmdlist
.cleanup_dir()
1336 class Command(object):
1337 """A command run in the build process."""
1339 def __init__(self
, desc
, num
, dir, path
, command
, always_run
=False):
1340 """Initialize a Command object."""
1344 trans
= str.maketrans({' ': '-'})
1345 self
.logbase
= '%03d-%s' % (num
, desc
.translate(trans
))
1346 self
.command
= command
1347 self
.always_run
= always_run
1350 def shell_make_quote_string(s
):
1351 """Given a string not containing a newline, quote it for use by the
1353 assert '\n' not in s
1354 if re
.fullmatch('[]+,./0-9@A-Z_a-z-]+', s
):
1356 strans
= str.maketrans({"'": "'\\''"})
1357 s
= "'%s'" % s
.translate(strans
)
1358 mtrans
= str.maketrans({'$': '$$'})
1359 return s
.translate(mtrans
)
1362 def shell_make_quote_list(l
, translate_make
):
1363 """Given a list of strings not containing newlines, quote them for use
1364 by the shell and make, returning a single string. If translate_make
1365 is true and the first string is 'make', change it to $(MAKE)."""
1366 l
= [Command
.shell_make_quote_string(s
) for s
in l
]
1367 if translate_make
and l
[0] == 'make':
1371 def shell_make_quote(self
):
1372 """Return this command quoted for the shell and make."""
1373 return self
.shell_make_quote_list(self
.command
, True)
1376 class CommandList(object):
1377 """A list of commands run in the build process."""
1379 def __init__(self
, desc
, keep
):
1380 """Initialize a CommandList object."""
1387 def desc_txt(self
, desc
):
1388 """Return the description to use for a command."""
1389 return '%s %s' % (' '.join(self
.desc
), desc
)
1391 def use_dir(self
, dir):
1392 """Set the default directory for subsequent commands."""
1395 def use_path(self
, path
):
1396 """Set a directory to be prepended to the PATH for subsequent
1400 def push_subdesc(self
, subdesc
):
1401 """Set the default subdescription for subsequent commands (e.g., the
1402 name of a component being built, within the series of commands
1404 self
.desc
.append(subdesc
)
1406 def pop_subdesc(self
):
1407 """Pop a subdescription from the list of descriptions."""
1410 def create_use_dir(self
, dir):
1411 """Remove and recreate a directory and use it for subsequent
1413 self
.add_command_dir('rm', None, ['rm', '-rf', dir])
1414 self
.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1417 def create_copy_dir(self
, src
, dest
):
1418 """Remove a directory and recreate it as a copy from the given
1420 self
.add_command_dir('copy-rm', None, ['rm', '-rf', dest
])
1421 parent
= os
.path
.dirname(dest
)
1422 self
.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent
])
1423 self
.add_command_dir('copy', None, ['cp', '-a', src
, dest
])
1425 def add_command_dir(self
, desc
, dir, command
, always_run
=False):
1426 """Add a command to run in a given directory."""
1427 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), dir, self
.path
,
1428 command
, always_run
)
1429 self
.cmdlist
.append(cmd
)
1431 def add_command(self
, desc
, command
, always_run
=False):
1432 """Add a command to run in the default directory."""
1433 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), self
.dir,
1434 self
.path
, command
, always_run
)
1435 self
.cmdlist
.append(cmd
)
1437 def cleanup_dir(self
, desc
='cleanup', dir=None):
1438 """Clean up a build directory. If no directory is specified, the
1439 default directory is cleaned up and ceases to be the default
1444 if self
.keep
!= 'all':
1445 self
.add_command_dir(desc
, None, ['rm', '-rf', dir],
1446 always_run
=(self
.keep
== 'none'))
1448 def makefile_commands(self
, wrapper
, logsdir
):
1449 """Return the sequence of commands in the form of text for a Makefile.
1450 The given wrapper script takes arguments: base of logs for
1451 previous command, or empty; base of logs for this command;
1452 description; directory; PATH addition; the command itself."""
1453 # prev_base is the base of the name for logs of the previous
1454 # command that is not always-run (that is, a build command,
1455 # whose failure should stop subsequent build commands from
1456 # being run, as opposed to a cleanup command, which is run
1457 # even if previous commands failed).
1460 for c
in self
.cmdlist
:
1461 ctxt
= c
.shell_make_quote()
1462 if prev_base
and not c
.always_run
:
1463 prev_log
= os
.path
.join(logsdir
, prev_base
)
1466 this_log
= os
.path
.join(logsdir
, c
.logbase
)
1467 if not c
.always_run
:
1468 prev_base
= c
.logbase
1477 prelims
= [wrapper
, prev_log
, this_log
, c
.desc
, dir, path
]
1478 prelim_txt
= Command
.shell_make_quote_list(prelims
, False)
1479 cmds
.append('\t@%s %s' % (prelim_txt
, ctxt
))
1480 return '\n'.join(cmds
)
1482 def status_logs(self
, logsdir
):
1483 """Return the list of log files with command status."""
1484 return [os
.path
.join(logsdir
, '%s-status.txt' % c
.logbase
)
1485 for c
in self
.cmdlist
]
1489 """Return an argument parser for this module."""
1490 parser
= argparse
.ArgumentParser(description
=__doc__
)
1491 parser
.add_argument('-j', dest
='parallelism',
1492 help='Run this number of jobs in parallel',
1493 type=int, default
=os
.cpu_count())
1494 parser
.add_argument('--keep', dest
='keep',
1495 help='Whether to keep all build directories, '
1496 'none or only those from failed builds',
1497 default
='none', choices
=('none', 'all', 'failed'))
1498 parser
.add_argument('--replace-sources', action
='store_true',
1499 help='Remove and replace source directories '
1500 'with the wrong version of a component')
1501 parser
.add_argument('--strip', action
='store_true',
1502 help='Strip installed glibc libraries')
1503 parser
.add_argument('topdir',
1504 help='Toplevel working directory')
1505 parser
.add_argument('action',
1507 choices
=('checkout', 'bot-cycle', 'bot',
1508 'host-libraries', 'compilers', 'glibcs'))
1509 parser
.add_argument('configs',
1510 help='Versions to check out or configurations to build',
1516 """The main entry point."""
1517 parser
= get_parser()
1518 opts
= parser
.parse_args(argv
)
1519 topdir
= os
.path
.abspath(opts
.topdir
)
1520 ctx
= Context(topdir
, opts
.parallelism
, opts
.keep
, opts
.replace_sources
,
1521 opts
.strip
, opts
.action
)
1522 ctx
.run_builds(opts
.action
, opts
.configs
)
1525 if __name__
== '__main__':