tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) / options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options):
  45     """Display a list of actions that we would take, if not a dry run.
  46
  47     Args:
  48         series: Series object
  49         why_selected: Dictionary where each key is a buildman argument
  50                 provided by the user, and the value is the list of boards
  51                 brought in by that argument. For example, 'arm' might bring
  52                 in 400 boards, so in this case the key would be 'arm' and
  53                 the value would be a list of board names.
  54         boards_selected: Dict of selected boards, key is target name,
  55                 value is Board object
  56         builder: The builder that will be used to build the commits
  57         options: Command line options object
  58     """
  59     col = terminal.Color()
  60     print 'Dry run, so not doing much. But I would do this:'
  61     print
  62     if series:
  63         commits = series.commits
  64     else:
  65         commits = None
  66     print GetActionSummary(False, commits, boards_selected,
  67             options)
  68     print 'Build directory: %s' % builder.base_dir
  69     if commits:
  70         for upto in range(0, len(series.commits), options.step):
  71             commit = series.commits[upto]
  72             print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
  73             print commit.subject
  74     print
  75     for arg in why_selected:
  76         if arg != 'all':
  77             print arg, ': %d boards' % len(why_selected[arg])
  78             if options.verbose:
  79                 print '   %s' % ' '.join(why_selected[arg])
  80     print ('Total boards to build for each commit: %d\n' %
  81             len(why_selected['all']))
  82
  83 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
  84                clean_dir=False):
  85     """The main control code for buildman
  86
  87     Args:
  88         options: Command line options object
  89         args: Command line arguments (list of strings)
  90         toolchains: Toolchains to use - this should be a Toolchains()
  91                 object. If None, then it will be created and scanned
  92         make_func: Make function to use for the builder. This is called
  93                 to execute 'make'. If this is None, the normal function
  94                 will be used, which calls the 'make' tool with suitable
  95                 arguments. This setting is useful for tests.
  96         board: Boards() object to use, containing a list of available
  97                 boards. If this is None it will be created and scanned.
  98     """
  99     global builder
 100
 101     if options.full_help:
 102         pager = os.getenv('PAGER')
 103         if not pager:
 104             pager = 'more'
 105         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 106                              'README')
 107         command.Run(pager, fname)
 108         return 0
 109
 110     gitutil.Setup()
 111     col = terminal.Color()
 112
 113     options.git_dir = os.path.join(options.git, '.git')
 114
 115     no_toolchains = toolchains is None
 116     if no_toolchains:
 117         toolchains = toolchain.Toolchains()
 118
 119     if options.fetch_arch:
 120         if options.fetch_arch == 'list':
 121             sorted_list = toolchains.ListArchs()
 122             print col.Color(col.BLUE, 'Available architectures: %s\n' %
 123                             ' '.join(sorted_list))
 124             return 0
 125         else:
 126             fetch_arch = options.fetch_arch
 127             if fetch_arch == 'all':
 128                 fetch_arch = ','.join(toolchains.ListArchs())
 129                 print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 130                                 fetch_arch)
 131             for arch in fetch_arch.split(','):
 132                 print
 133                 ret = toolchains.FetchAndInstall(arch)
 134                 if ret:
 135                     return ret
 136             return 0
 137
 138     if no_toolchains:
 139         toolchains.GetSettings()
 140         toolchains.Scan(options.list_tool_chains)
 141     if options.list_tool_chains:
 142         toolchains.List()
 143         print
 144         return 0
 145
 146     # Work out how many commits to build. We want to build everything on the
 147     # branch. We also build the upstream commit as a control so we can see
 148     # problems introduced by the first commit on the branch.
 149     count = options.count
 150     has_range = options.branch and '..' in options.branch
 151     if count == -1:
 152         if not options.branch:
 153             count = 1
 154         else:
 155             if has_range:
 156                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 157                                                          options.branch)
 158             else:
 159                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 160                                                           options.branch)
 161             if count is None:
 162                 sys.exit(col.Color(col.RED, msg))
 163             elif count == 0:
 164                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 165                                    options.branch))
 166             if msg:
 167                 print col.Color(col.YELLOW, msg)
 168             count += 1   # Build upstream commit also
 169
 170     if not count:
 171         str = ("No commits found to process in branch '%s': "
 172                "set branch's upstream or use -c flag" % options.branch)
 173         sys.exit(col.Color(col.RED, str))
 174
 175     # Work out what subset of the boards we are building
 176     if not boards:
 177         board_file = os.path.join(options.git, 'boards.cfg')
 178         status = subprocess.call([os.path.join(options.git,
 179                                                 'tools/genboardscfg.py')])
 180         if status != 0:
 181                 sys.exit("Failed to generate boards.cfg")
 182
 183         boards = board.Boards()
 184         boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 185
 186     exclude = []
 187     if options.exclude:
 188         for arg in options.exclude:
 189             exclude += arg.split(',')
 190
 191     why_selected = boards.SelectBoards(args, exclude)
 192     selected = boards.GetSelected()
 193     if not len(selected):
 194         sys.exit(col.Color(col.RED, 'No matching boards found'))
 195
 196     # Read the metadata from the commits. First look at the upstream commit,
 197     # then the ones in the branch. We would like to do something like
 198     # upstream/master~..branch but that isn't possible if upstream/master is
 199     # a merge commit (it will list all the commits that form part of the
 200     # merge)
 201     # Conflicting tags are not a problem for buildman, since it does not use
 202     # them. For example, Series-version is not useful for buildman. On the
 203     # other hand conflicting tags will cause an error. So allow later tags
 204     # to overwrite earlier ones by setting allow_overwrite=True
 205     if options.branch:
 206         if count == -1:
 207             if has_range:
 208                 range_expr = options.branch
 209             else:
 210                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 211                                                       options.branch)
 212             upstream_commit = gitutil.GetUpstream(options.git_dir,
 213                                                   options.branch)
 214             series = patchstream.GetMetaDataForList(upstream_commit,
 215                 options.git_dir, 1, series=None, allow_overwrite=True)
 216
 217             series = patchstream.GetMetaDataForList(range_expr,
 218                     options.git_dir, None, series, allow_overwrite=True)
 219         else:
 220             # Honour the count
 221             series = patchstream.GetMetaDataForList(options.branch,
 222                     options.git_dir, count, series=None, allow_overwrite=True)
 223     else:
 224         series = None
 225         if not options.dry_run:
 226             options.verbose = True
 227             if not options.summary:
 228                 options.show_errors = True
 229
 230     # By default we have one thread per CPU. But if there are not enough jobs
 231     # we can have fewer threads and use a high '-j' value for make.
 232     if not options.threads:
 233         options.threads = min(multiprocessing.cpu_count(), len(selected))
 234     if not options.jobs:
 235         options.jobs = max(1, (multiprocessing.cpu_count() +
 236                 len(selected) - 1) / len(selected))
 237
 238     if not options.step:
 239         options.step = len(series.commits) - 1
 240
 241     gnu_make = command.Output(os.path.join(options.git,
 242             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 243     if not gnu_make:
 244         sys.exit('GNU Make not found')
 245
 246     # Create a new builder with the selected options.
 247     output_dir = options.output_dir
 248     if options.branch:
 249         dirname = options.branch.replace('/', '_')
 250         # As a special case allow the board directory to be placed in the
 251         # output directory itself rather than any subdirectory.
 252         if not options.no_subdirs:
 253             output_dir = os.path.join(options.output_dir, dirname)
 254     if (clean_dir and output_dir != options.output_dir and
 255             os.path.exists(output_dir)):
 256         shutil.rmtree(output_dir)
 257     builder = Builder(toolchains, output_dir, options.git_dir,
 258             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 259             show_unknown=options.show_unknown, step=options.step,
 260             no_subdirs=options.no_subdirs, full_path=options.full_path,
 261             verbose_build=options.verbose_build,
 262             incremental=options.incremental,
 263             per_board_out_dir=options.per_board_out_dir,
 264             config_only=options.config_only,
 265             squash_config_y=not options.preserve_config_y,
 266             warnings_as_errors=options.warnings_as_errors)
 267     builder.force_config_on_failure = not options.quick
 268     if make_func:
 269         builder.do_make = make_func
 270
 271     # For a dry run, just show our actions as a sanity check
 272     if options.dry_run:
 273         ShowActions(series, why_selected, selected, builder, options)
 274     else:
 275         builder.force_build = options.force_build
 276         builder.force_build_failures = options.force_build_failures
 277         builder.force_reconfig = options.force_reconfig
 278         builder.in_tree = options.in_tree
 279
 280         # Work out which boards to build
 281         board_selected = boards.GetSelectedDict()
 282
 283         if series:
 284             commits = series.commits
 285             # Number the commits for test purposes
 286             for commit in range(len(commits)):
 287                 commits[commit].sequence = commit
 288         else:
 289             commits = None
 290
 291         Print(GetActionSummary(options.summary, commits, board_selected,
 292                                 options))
 293
 294         # We can't show function sizes without board details at present
 295         if options.show_bloat:
 296             options.show_detail = True
 297         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 298                                   options.show_detail, options.show_bloat,
 299                                   options.list_error_boards,
 300                                   options.show_config)
 301         if options.summary:
 302             builder.ShowSummary(commits, board_selected)
 303         else:
 304             fail, warned = builder.BuildBoards(commits, board_selected,
 305                                 options.keep_outputs, options.verbose)
 306             if fail:
 307                 return 128
 308             elif warned:
 309                 return 129
 310     return 0