tools/buildman/builderthread.py

   1 # Copyright (c) 2014 Google, Inc
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import errno
   7 import glob
   8 import os
   9 import shutil
  10 import threading
  11
  12 import command
  13 import gitutil
  14
  15 RETURN_CODE_RETRY = -1
  16
  17 def Mkdir(dirname, parents = False):
  18     """Make a directory if it doesn't already exist.
  19
  20     Args:
  21         dirname: Directory to create
  22     """
  23     try:
  24         if parents:
  25             os.makedirs(dirname)
  26         else:
  27             os.mkdir(dirname)
  28     except OSError as err:
  29         if err.errno == errno.EEXIST:
  30             pass
  31         else:
  32             raise
  33
  34 class BuilderJob:
  35     """Holds information about a job to be performed by a thread
  36
  37     Members:
  38         board: Board object to build
  39         commits: List of commit options to build.
  40     """
  41     def __init__(self):
  42         self.board = None
  43         self.commits = []
  44
  45
  46 class ResultThread(threading.Thread):
  47     """This thread processes results from builder threads.
  48
  49     It simply passes the results on to the builder. There is only one
  50     result thread, and this helps to serialise the build output.
  51     """
  52     def __init__(self, builder):
  53         """Set up a new result thread
  54
  55         Args:
  56             builder: Builder which will be sent each result
  57         """
  58         threading.Thread.__init__(self)
  59         self.builder = builder
  60
  61     def run(self):
  62         """Called to start up the result thread.
  63
  64         We collect the next result job and pass it on to the build.
  65         """
  66         while True:
  67             result = self.builder.out_queue.get()
  68             self.builder.ProcessResult(result)
  69             self.builder.out_queue.task_done()
  70
  71
  72 class BuilderThread(threading.Thread):
  73     """This thread builds U-Boot for a particular board.
  74
  75     An input queue provides each new job. We run 'make' to build U-Boot
  76     and then pass the results on to the output queue.
  77
  78     Members:
  79         builder: The builder which contains information we might need
  80         thread_num: Our thread number (0-n-1), used to decide on a
  81                 temporary directory
  82     """
  83     def __init__(self, builder, thread_num):
  84         """Set up a new builder thread"""
  85         threading.Thread.__init__(self)
  86         self.builder = builder
  87         self.thread_num = thread_num
  88
  89     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  90         """Run 'make' on a particular commit and board.
  91
  92         The source code will already be checked out, so the 'commit'
  93         argument is only for information.
  94
  95         Args:
  96             commit: Commit object that is being built
  97             brd: Board object that is being built
  98             stage: Stage of the build. Valid stages are:
  99                         mrproper - can be called to clean source
 100                         config - called to configure for a board
 101                         build - the main make invocation - it does the build
 102             args: A list of arguments to pass to 'make'
 103             kwargs: A list of keyword arguments to pass to command.RunPipe()
 104
 105         Returns:
 106             CommandResult object
 107         """
 108         return self.builder.do_make(commit, brd, stage, cwd, *args,
 109                 **kwargs)
 110
 111     def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
 112                   force_build_failures):
 113         """Build a particular commit.
 114
 115         If the build is already done, and we are not forcing a build, we skip
 116         the build and just return the previously-saved results.
 117
 118         Args:
 119             commit_upto: Commit number to build (0...n-1)
 120             brd: Board object to build
 121             work_dir: Directory to which the source will be checked out
 122             do_config: True to run a make <board>_defconfig on the source
 123             force_build: Force a build even if one was previously done
 124             force_build_failures: Force a bulid if the previous result showed
 125                 failure
 126
 127         Returns:
 128             tuple containing:
 129                 - CommandResult object containing the results of the build
 130                 - boolean indicating whether 'make config' is still needed
 131         """
 132         # Create a default result - it will be overwritte by the call to
 133         # self.Make() below, in the event that we do a build.
 134         result = command.CommandResult()
 135         result.return_code = 0
 136         if self.builder.in_tree:
 137             out_dir = work_dir
 138         else:
 139             out_dir = os.path.join(work_dir, 'build')
 140
 141         # Check if the job was already completed last time
 142         done_file = self.builder.GetDoneFile(commit_upto, brd.target)
 143         result.already_done = os.path.exists(done_file)
 144         will_build = (force_build or force_build_failures or
 145             not result.already_done)
 146         if result.already_done:
 147             # Get the return code from that build and use it
 148             with open(done_file, 'r') as fd:
 149                 result.return_code = int(fd.readline())
 150
 151             # Check the signal that the build needs to be retried
 152             if result.return_code == RETURN_CODE_RETRY:
 153                 will_build = True
 154             elif will_build:
 155                 err_file = self.builder.GetErrFile(commit_upto, brd.target)
 156                 if os.path.exists(err_file) and os.stat(err_file).st_size:
 157                     result.stderr = 'bad'
 158                 elif not force_build:
 159                     # The build passed, so no need to build it again
 160                     will_build = False
 161
 162         if will_build:
 163             # We are going to have to build it. First, get a toolchain
 164             if not self.toolchain:
 165                 try:
 166                     self.toolchain = self.builder.toolchains.Select(brd.arch)
 167                 except ValueError as err:
 168                     result.return_code = 10
 169                     result.stdout = ''
 170                     result.stderr = str(err)
 171                     # TODO(sjg@chromium.org): This gets swallowed, but needs
 172                     # to be reported.
 173
 174             if self.toolchain:
 175                 # Checkout the right commit
 176                 if self.builder.commits:
 177                     commit = self.builder.commits[commit_upto]
 178                     if self.builder.checkout:
 179                         git_dir = os.path.join(work_dir, '.git')
 180                         gitutil.Checkout(commit.hash, git_dir, work_dir,
 181                                          force=True)
 182                 else:
 183                     commit = 'current'
 184
 185                 # Set up the environment and command line
 186                 env = self.toolchain.MakeEnvironment(self.builder.full_path)
 187                 Mkdir(out_dir)
 188                 args = []
 189                 cwd = work_dir
 190                 src_dir = os.path.realpath(work_dir)
 191                 if not self.builder.in_tree:
 192                     if commit_upto is None:
 193                         # In this case we are building in the original source
 194                         # directory (i.e. the current directory where buildman
 195                         # is invoked. The output directory is set to this
 196                         # thread's selected work directory.
 197                         #
 198                         # Symlinks can confuse U-Boot's Makefile since
 199                         # we may use '..' in our path, so remove them.
 200                         work_dir = os.path.realpath(work_dir)
 201                         args.append('O=%s/build' % work_dir)
 202                         cwd = None
 203                         src_dir = os.getcwd()
 204                     else:
 205                         args.append('O=build')
 206                 if self.builder.verbose_build:
 207                     args.append('V=1')
 208                 else:
 209                     args.append('-s')
 210                 if self.builder.num_jobs is not None:
 211                     args.extend(['-j', str(self.builder.num_jobs)])
 212                 config_args = ['%s_defconfig' % brd.target]
 213                 config_out = ''
 214                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
 215
 216                 # If we need to reconfigure, do that now
 217                 if do_config:
 218                     result = self.Make(commit, brd, 'mrproper', cwd,
 219                             'mrproper', *args, env=env)
 220                     config_out = result.combined
 221                     result = self.Make(commit, brd, 'config', cwd,
 222                             *(args + config_args), env=env)
 223                     config_out += result.combined
 224                     do_config = False   # No need to configure next time
 225                 if result.return_code == 0:
 226                     result = self.Make(commit, brd, 'build', cwd, *args,
 227                             env=env)
 228                 result.stderr = result.stderr.replace(src_dir + '/', '')
 229                 if self.builder.verbose_build:
 230                     result.stdout = config_out + result.stdout
 231             else:
 232                 result.return_code = 1
 233                 result.stderr = 'No tool chain for %s\n' % brd.arch
 234             result.already_done = False
 235
 236         result.toolchain = self.toolchain
 237         result.brd = brd
 238         result.commit_upto = commit_upto
 239         result.out_dir = out_dir
 240         return result, do_config
 241
 242     def _WriteResult(self, result, keep_outputs):
 243         """Write a built result to the output directory.
 244
 245         Args:
 246             result: CommandResult object containing result to write
 247             keep_outputs: True to store the output binaries, False
 248                 to delete them
 249         """
 250         # Fatal error
 251         if result.return_code < 0:
 252             return
 253
 254         # If we think this might have been aborted with Ctrl-C, record the
 255         # failure but not that we are 'done' with this board. A retry may fix
 256         # it.
 257         maybe_aborted =  result.stderr and 'No child processes' in result.stderr
 258
 259         if result.already_done:
 260             return
 261
 262         # Write the output and stderr
 263         output_dir = self.builder._GetOutputDir(result.commit_upto)
 264         Mkdir(output_dir)
 265         build_dir = self.builder.GetBuildDir(result.commit_upto,
 266                 result.brd.target)
 267         Mkdir(build_dir)
 268
 269         outfile = os.path.join(build_dir, 'log')
 270         with open(outfile, 'w') as fd:
 271             if result.stdout:
 272                 fd.write(result.stdout)
 273
 274         errfile = self.builder.GetErrFile(result.commit_upto,
 275                 result.brd.target)
 276         if result.stderr:
 277             with open(errfile, 'w') as fd:
 278                 fd.write(result.stderr)
 279         elif os.path.exists(errfile):
 280             os.remove(errfile)
 281
 282         if result.toolchain:
 283             # Write the build result and toolchain information.
 284             done_file = self.builder.GetDoneFile(result.commit_upto,
 285                     result.brd.target)
 286             with open(done_file, 'w') as fd:
 287                 if maybe_aborted:
 288                     # Special code to indicate we need to retry
 289                     fd.write('%s' % RETURN_CODE_RETRY)
 290                 else:
 291                     fd.write('%s' % result.return_code)
 292             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
 293                 print >>fd, 'gcc', result.toolchain.gcc
 294                 print >>fd, 'path', result.toolchain.path
 295                 print >>fd, 'cross', result.toolchain.cross
 296                 print >>fd, 'arch', result.toolchain.arch
 297                 fd.write('%s' % result.return_code)
 298
 299             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
 300                 print >>fd, 'gcc', result.toolchain.gcc
 301                 print >>fd, 'path', result.toolchain.path
 302
 303             # Write out the image and function size information and an objdump
 304             env = result.toolchain.MakeEnvironment(self.builder.full_path)
 305             lines = []
 306             for fname in ['u-boot', 'spl/u-boot-spl']:
 307                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
 308                 nm_result = command.RunPipe([cmd], capture=True,
 309                         capture_stderr=True, cwd=result.out_dir,
 310                         raise_on_error=False, env=env)
 311                 if nm_result.stdout:
 312                     nm = self.builder.GetFuncSizesFile(result.commit_upto,
 313                                     result.brd.target, fname)
 314                     with open(nm, 'w') as fd:
 315                         print >>fd, nm_result.stdout,
 316
 317                 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
 318                 dump_result = command.RunPipe([cmd], capture=True,
 319                         capture_stderr=True, cwd=result.out_dir,
 320                         raise_on_error=False, env=env)
 321                 rodata_size = ''
 322                 if dump_result.stdout:
 323                     objdump = self.builder.GetObjdumpFile(result.commit_upto,
 324                                     result.brd.target, fname)
 325                     with open(objdump, 'w') as fd:
 326                         print >>fd, dump_result.stdout,
 327                     for line in dump_result.stdout.splitlines():
 328                         fields = line.split()
 329                         if len(fields) > 5 and fields[1] == '.rodata':
 330                             rodata_size = fields[2]
 331
 332                 cmd = ['%ssize' % self.toolchain.cross, fname]
 333                 size_result = command.RunPipe([cmd], capture=True,
 334                         capture_stderr=True, cwd=result.out_dir,
 335                         raise_on_error=False, env=env)
 336                 if size_result.stdout:
 337                     lines.append(size_result.stdout.splitlines()[1] + ' ' +
 338                                  rodata_size)
 339
 340             # Write out the image sizes file. This is similar to the output
 341             # of binutil's 'size' utility, but it omits the header line and
 342             # adds an additional hex value at the end of each line for the
 343             # rodata size
 344             if len(lines):
 345                 sizes = self.builder.GetSizesFile(result.commit_upto,
 346                                 result.brd.target)
 347                 with open(sizes, 'w') as fd:
 348                     print >>fd, '\n'.join(lines)
 349
 350         # Write out the configuration files, with a special case for SPL
 351         for dirname in ['', 'spl', 'tpl']:
 352             self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
 353                 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
 354                 'include/autoconf.mk', 'include/generated/autoconf.h'])
 355
 356         # Now write the actual build output
 357         if keep_outputs:
 358             self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
 359                 '*.map', '*.img', 'MLO', 'include/autoconf.mk',
 360                 'spl/u-boot-spl*'])
 361
 362     def CopyFiles(self, out_dir, build_dir, dirname, patterns):
 363         """Copy files from the build directory to the output.
 364
 365         Args:
 366             out_dir: Path to output directory containing the files
 367             build_dir: Place to copy the files
 368             dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
 369             patterns: A list of filenames (strings) to copy, each relative
 370                to the build directory
 371         """
 372         for pattern in patterns:
 373             file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
 374             for fname in file_list:
 375                 target = os.path.basename(fname)
 376                 if dirname:
 377                     base, ext = os.path.splitext(target)
 378                     if ext:
 379                         target = '%s-%s%s' % (base, dirname, ext)
 380                 shutil.copy(fname, os.path.join(build_dir, target))
 381
 382     def RunJob(self, job):
 383         """Run a single job
 384
 385         A job consists of a building a list of commits for a particular board.
 386
 387         Args:
 388             job: Job to build
 389         """
 390         brd = job.board
 391         work_dir = self.builder.GetThreadDir(self.thread_num)
 392         self.toolchain = None
 393         if job.commits:
 394             # Run 'make board_defconfig' on the first commit
 395             do_config = True
 396             commit_upto  = 0
 397             force_build = False
 398             for commit_upto in range(0, len(job.commits), job.step):
 399                 result, request_config = self.RunCommit(commit_upto, brd,
 400                         work_dir, do_config,
 401                         force_build or self.builder.force_build,
 402                         self.builder.force_build_failures)
 403                 failed = result.return_code or result.stderr
 404                 did_config = do_config
 405                 if failed and not do_config:
 406                     # If our incremental build failed, try building again
 407                     # with a reconfig.
 408                     if self.builder.force_config_on_failure:
 409                         result, request_config = self.RunCommit(commit_upto,
 410                             brd, work_dir, True, True, False)
 411                         did_config = True
 412                 if not self.builder.force_reconfig:
 413                     do_config = request_config
 414
 415                 # If we built that commit, then config is done. But if we got
 416                 # an warning, reconfig next time to force it to build the same
 417                 # files that created warnings this time. Otherwise an
 418                 # incremental build may not build the same file, and we will
 419                 # think that the warning has gone away.
 420                 # We could avoid this by using -Werror everywhere...
 421                 # For errors, the problem doesn't happen, since presumably
 422                 # the build stopped and didn't generate output, so will retry
 423                 # that file next time. So we could detect warnings and deal
 424                 # with them specially here. For now, we just reconfigure if
 425                 # anything goes work.
 426                 # Of course this is substantially slower if there are build
 427                 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
 428                 # have problems).
 429                 if (failed and not result.already_done and not did_config and
 430                         self.builder.force_config_on_failure):
 431                     # If this build failed, try the next one with a
 432                     # reconfigure.
 433                     # Sometimes if the board_config.h file changes it can mess
 434                     # with dependencies, and we get:
 435                     # make: *** No rule to make target `include/autoconf.mk',
 436                     #     needed by `depend'.
 437                     do_config = True
 438                     force_build = True
 439                 else:
 440                     force_build = False
 441                     if self.builder.force_config_on_failure:
 442                         if failed:
 443                             do_config = True
 444                     result.commit_upto = commit_upto
 445                     if result.return_code < 0:
 446                         raise ValueError('Interrupt')
 447
 448                 # We have the build results, so output the result
 449                 self._WriteResult(result, job.keep_outputs)
 450                 self.builder.out_queue.put(result)
 451         else:
 452             # Just build the currently checked-out build
 453             result, request_config = self.RunCommit(None, brd, work_dir, True,
 454                         True, self.builder.force_build_failures)
 455             result.commit_upto = 0
 456             self._WriteResult(result, job.keep_outputs)
 457             self.builder.out_queue.put(result)
 458
 459     def run(self):
 460         """Our thread's run function
 461
 462         This thread picks a job from the queue, runs it, and then goes to the
 463         next job.
 464         """
 465         alive = True
 466         while True:
 467             job = self.builder.queue.get()
 468             if self.builder.active and alive:
 469                 self.RunJob(job)
 470             '''
 471             try:
 472                 if self.builder.active and alive:
 473                     self.RunJob(job)
 474             except Exception as err:
 475                 alive = False
 476                 print err
 477             '''
 478             self.builder.queue.task_done()