tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import shutil
   9 import sys
  10
  11 import board
  12 import bsettings
  13 from builder import Builder
  14 import gitutil
  15 import patchstream
  16 import terminal
  17 from terminal import Print
  18 import toolchain
  19 import command
  20 import subprocess
  21
  22 def GetPlural(count):
  23     """Returns a plural 's' if count is not 1"""
  24     return 's' if count != 1 else ''
  25
  26 def GetActionSummary(is_summary, commits, selected, options):
  27     """Return a string summarising the intended action.
  28
  29     Returns:
  30         Summary string.
  31     """
  32     if commits:
  33         count = len(commits)
  34         count = (count + options.step - 1) / options.step
  35         commit_str = '%d commit%s' % (count, GetPlural(count))
  36     else:
  37         commit_str = 'current source'
  38     str = '%s %s for %d boards' % (
  39         'Summary of' if is_summary else 'Building', commit_str,
  40         len(selected))
  41     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  42             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  43     return str
  44
  45 def ShowActions(series, why_selected, boards_selected, builder, options):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the boards brought
  52                 in by that argument. For example, 'arm' might bring in
  53                 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59     """
  60     col = terminal.Color()
  61     print 'Dry run, so not doing much. But I would do this:'
  62     print
  63     if series:
  64         commits = series.commits
  65     else:
  66         commits = None
  67     print GetActionSummary(False, commits, boards_selected,
  68             options)
  69     print 'Build directory: %s' % builder.base_dir
  70     if commits:
  71         for upto in range(0, len(series.commits), options.step):
  72             commit = series.commits[upto]
  73             print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
  74             print commit.subject
  75     print
  76     for arg in why_selected:
  77         if arg != 'all':
  78             print arg, ': %d boards' % why_selected[arg]
  79     print ('Total boards to build for each commit: %d\n' %
  80             why_selected['all'])
  81
  82 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
  83                clean_dir=False):
  84     """The main control code for buildman
  85
  86     Args:
  87         options: Command line options object
  88         args: Command line arguments (list of strings)
  89         toolchains: Toolchains to use - this should be a Toolchains()
  90                 object. If None, then it will be created and scanned
  91         make_func: Make function to use for the builder. This is called
  92                 to execute 'make'. If this is None, the normal function
  93                 will be used, which calls the 'make' tool with suitable
  94                 arguments. This setting is useful for tests.
  95         board: Boards() object to use, containing a list of available
  96                 boards. If this is None it will be created and scanned.
  97     """
  98     global builder
  99
 100     if options.full_help:
 101         pager = os.getenv('PAGER')
 102         if not pager:
 103             pager = 'more'
 104         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 105                              'README')
 106         command.Run(pager, fname)
 107         return 0
 108
 109     gitutil.Setup()
 110     col = terminal.Color()
 111
 112     options.git_dir = os.path.join(options.git, '.git')
 113
 114     no_toolchains = toolchains is None
 115     if no_toolchains:
 116         toolchains = toolchain.Toolchains()
 117
 118     if options.fetch_arch:
 119         if options.fetch_arch == 'list':
 120             sorted_list = toolchains.ListArchs()
 121             print col.Color(col.BLUE, 'Available architectures: %s\n' %
 122                             ' '.join(sorted_list))
 123             return 0
 124         else:
 125             fetch_arch = options.fetch_arch
 126             if fetch_arch == 'all':
 127                 fetch_arch = ','.join(toolchains.ListArchs())
 128                 print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 129                                 fetch_arch)
 130             for arch in fetch_arch.split(','):
 131                 print
 132                 ret = toolchains.FetchAndInstall(arch)
 133                 if ret:
 134                     return ret
 135             return 0
 136
 137     if no_toolchains:
 138         toolchains.GetSettings()
 139         toolchains.Scan(options.list_tool_chains)
 140     if options.list_tool_chains:
 141         toolchains.List()
 142         print
 143         return 0
 144
 145     # Work out how many commits to build. We want to build everything on the
 146     # branch. We also build the upstream commit as a control so we can see
 147     # problems introduced by the first commit on the branch.
 148     count = options.count
 149     has_range = options.branch and '..' in options.branch
 150     if count == -1:
 151         if not options.branch:
 152             count = 1
 153         else:
 154             if has_range:
 155                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 156                                                          options.branch)
 157             else:
 158                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 159                                                           options.branch)
 160             if count is None:
 161                 sys.exit(col.Color(col.RED, msg))
 162             elif count == 0:
 163                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 164                                    options.branch))
 165             if msg:
 166                 print col.Color(col.YELLOW, msg)
 167             count += 1   # Build upstream commit also
 168
 169     if not count:
 170         str = ("No commits found to process in branch '%s': "
 171                "set branch's upstream or use -c flag" % options.branch)
 172         sys.exit(col.Color(col.RED, str))
 173
 174     # Work out what subset of the boards we are building
 175     if not boards:
 176         board_file = os.path.join(options.git, 'boards.cfg')
 177         status = subprocess.call([os.path.join(options.git,
 178                                                 'tools/genboardscfg.py')])
 179         if status != 0:
 180                 sys.exit("Failed to generate boards.cfg")
 181
 182         boards = board.Boards()
 183         boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 184
 185     exclude = []
 186     if options.exclude:
 187         for arg in options.exclude:
 188             exclude += arg.split(',')
 189
 190     why_selected = boards.SelectBoards(args, exclude)
 191     selected = boards.GetSelected()
 192     if not len(selected):
 193         sys.exit(col.Color(col.RED, 'No matching boards found'))
 194
 195     # Read the metadata from the commits. First look at the upstream commit,
 196     # then the ones in the branch. We would like to do something like
 197     # upstream/master~..branch but that isn't possible if upstream/master is
 198     # a merge commit (it will list all the commits that form part of the
 199     # merge)
 200     # Conflicting tags are not a problem for buildman, since it does not use
 201     # them. For example, Series-version is not useful for buildman. On the
 202     # other hand conflicting tags will cause an error. So allow later tags
 203     # to overwrite earlier ones by setting allow_overwrite=True
 204     if options.branch:
 205         if count == -1:
 206             if has_range:
 207                 range_expr = options.branch
 208             else:
 209                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 210                                                       options.branch)
 211             upstream_commit = gitutil.GetUpstream(options.git_dir,
 212                                                   options.branch)
 213             series = patchstream.GetMetaDataForList(upstream_commit,
 214                 options.git_dir, 1, series=None, allow_overwrite=True)
 215
 216             series = patchstream.GetMetaDataForList(range_expr,
 217                     options.git_dir, None, series, allow_overwrite=True)
 218         else:
 219             # Honour the count
 220             series = patchstream.GetMetaDataForList(options.branch,
 221                     options.git_dir, count, series=None, allow_overwrite=True)
 222     else:
 223         series = None
 224         options.verbose = True
 225         if not options.summary:
 226             options.show_errors = True
 227
 228     # By default we have one thread per CPU. But if there are not enough jobs
 229     # we can have fewer threads and use a high '-j' value for make.
 230     if not options.threads:
 231         options.threads = min(multiprocessing.cpu_count(), len(selected))
 232     if not options.jobs:
 233         options.jobs = max(1, (multiprocessing.cpu_count() +
 234                 len(selected) - 1) / len(selected))
 235
 236     if not options.step:
 237         options.step = len(series.commits) - 1
 238
 239     gnu_make = command.Output(os.path.join(options.git,
 240             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 241     if not gnu_make:
 242         sys.exit('GNU Make not found')
 243
 244     # Create a new builder with the selected options.
 245     output_dir = options.output_dir
 246     if options.branch:
 247         dirname = options.branch.replace('/', '_')
 248         # As a special case allow the board directory to be placed in the
 249         # output directory itself rather than any subdirectory.
 250         if not options.no_subdirs:
 251             output_dir = os.path.join(options.output_dir, dirname)
 252     if (clean_dir and output_dir != options.output_dir and
 253             os.path.exists(output_dir)):
 254         shutil.rmtree(output_dir)
 255     builder = Builder(toolchains, output_dir, options.git_dir,
 256             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 257             show_unknown=options.show_unknown, step=options.step,
 258             no_subdirs=options.no_subdirs, full_path=options.full_path,
 259             verbose_build=options.verbose_build,
 260             incremental=options.incremental,
 261             per_board_out_dir=options.per_board_out_dir,)
 262     builder.force_config_on_failure = not options.quick
 263     if make_func:
 264         builder.do_make = make_func
 265
 266     # For a dry run, just show our actions as a sanity check
 267     if options.dry_run:
 268         ShowActions(series, why_selected, selected, builder, options)
 269     else:
 270         builder.force_build = options.force_build
 271         builder.force_build_failures = options.force_build_failures
 272         builder.force_reconfig = options.force_reconfig
 273         builder.in_tree = options.in_tree
 274
 275         # Work out which boards to build
 276         board_selected = boards.GetSelectedDict()
 277
 278         if series:
 279             commits = series.commits
 280             # Number the commits for test purposes
 281             for commit in range(len(commits)):
 282                 commits[commit].sequence = commit
 283         else:
 284             commits = None
 285
 286         Print(GetActionSummary(options.summary, commits, board_selected,
 287                                 options))
 288
 289         # We can't show function sizes without board details at present
 290         if options.show_bloat:
 291             options.show_detail = True
 292         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 293                                   options.show_detail, options.show_bloat,
 294                                   options.list_error_boards,
 295                                   options.show_config)
 296         if options.summary:
 297             builder.ShowSummary(commits, board_selected)
 298         else:
 299             fail, warned = builder.BuildBoards(commits, board_selected,
 300                                 options.keep_outputs, options.verbose)
 301             if fail:
 302                 return 128
 303             elif warned:
 304                 return 129
 305     return 0