tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import shutil
   9 import sys
  10
  11 import board
  12 import bsettings
  13 from builder import Builder
  14 import gitutil
  15 import patchstream
  16 import terminal
  17 from terminal import Print
  18 import toolchain
  19 import command
  20 import subprocess
  21
  22 def GetPlural(count):
  23     """Returns a plural 's' if count is not 1"""
  24     return 's' if count != 1 else ''
  25
  26 def GetActionSummary(is_summary, commits, selected, options):
  27     """Return a string summarising the intended action.
  28
  29     Returns:
  30         Summary string.
  31     """
  32     if commits:
  33         count = len(commits)
  34         count = (count + options.step - 1) / options.step
  35         commit_str = '%d commit%s' % (count, GetPlural(count))
  36     else:
  37         commit_str = 'current source'
  38     str = '%s %s for %d boards' % (
  39         'Summary of' if is_summary else 'Building', commit_str,
  40         len(selected))
  41     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  42             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  43     return str
  44
  45 def ShowActions(series, why_selected, boards_selected, builder, options):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the boards brought
  52                 in by that argument. For example, 'arm' might bring in
  53                 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59     """
  60     col = terminal.Color()
  61     print 'Dry run, so not doing much. But I would do this:'
  62     print
  63     if series:
  64         commits = series.commits
  65     else:
  66         commits = None
  67     print GetActionSummary(False, commits, boards_selected,
  68             options)
  69     print 'Build directory: %s' % builder.base_dir
  70     if commits:
  71         for upto in range(0, len(series.commits), options.step):
  72             commit = series.commits[upto]
  73             print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
  74             print commit.subject
  75     print
  76     for arg in why_selected:
  77         if arg != 'all':
  78             print arg, ': %d boards' % why_selected[arg]
  79     print ('Total boards to build for each commit: %d\n' %
  80             why_selected['all'])
  81
  82 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
  83                clean_dir=False):
  84     """The main control code for buildman
  85
  86     Args:
  87         options: Command line options object
  88         args: Command line arguments (list of strings)
  89         toolchains: Toolchains to use - this should be a Toolchains()
  90                 object. If None, then it will be created and scanned
  91         make_func: Make function to use for the builder. This is called
  92                 to execute 'make'. If this is None, the normal function
  93                 will be used, which calls the 'make' tool with suitable
  94                 arguments. This setting is useful for tests.
  95         board: Boards() object to use, containing a list of available
  96                 boards. If this is None it will be created and scanned.
  97     """
  98     global builder
  99
 100     if options.full_help:
 101         pager = os.getenv('PAGER')
 102         if not pager:
 103             pager = 'more'
 104         fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
 105         command.Run(pager, fname)
 106         return 0
 107
 108     gitutil.Setup()
 109
 110     options.git_dir = os.path.join(options.git, '.git')
 111
 112     if not toolchains:
 113         toolchains = toolchain.Toolchains()
 114         toolchains.GetSettings()
 115         toolchains.Scan(options.list_tool_chains)
 116     if options.list_tool_chains:
 117         toolchains.List()
 118         print
 119         return 0
 120
 121     # Work out how many commits to build. We want to build everything on the
 122     # branch. We also build the upstream commit as a control so we can see
 123     # problems introduced by the first commit on the branch.
 124     col = terminal.Color()
 125     count = options.count
 126     has_range = options.branch and '..' in options.branch
 127     if count == -1:
 128         if not options.branch:
 129             count = 1
 130         else:
 131             if has_range:
 132                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 133                                                          options.branch)
 134             else:
 135                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 136                                                           options.branch)
 137             if count is None:
 138                 sys.exit(col.Color(col.RED, msg))
 139             elif count == 0:
 140                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 141                                    options.branch))
 142             if msg:
 143                 print col.Color(col.YELLOW, msg)
 144             count += 1   # Build upstream commit also
 145
 146     if not count:
 147         str = ("No commits found to process in branch '%s': "
 148                "set branch's upstream or use -c flag" % options.branch)
 149         sys.exit(col.Color(col.RED, str))
 150
 151     # Work out what subset of the boards we are building
 152     if not boards:
 153         board_file = os.path.join(options.git, 'boards.cfg')
 154         status = subprocess.call([os.path.join(options.git,
 155                                                 'tools/genboardscfg.py')])
 156         if status != 0:
 157                 sys.exit("Failed to generate boards.cfg")
 158
 159         boards = board.Boards()
 160         boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 161
 162     exclude = []
 163     if options.exclude:
 164         for arg in options.exclude:
 165             exclude += arg.split(',')
 166
 167     why_selected = boards.SelectBoards(args, exclude)
 168     selected = boards.GetSelected()
 169     if not len(selected):
 170         sys.exit(col.Color(col.RED, 'No matching boards found'))
 171
 172     # Read the metadata from the commits. First look at the upstream commit,
 173     # then the ones in the branch. We would like to do something like
 174     # upstream/master~..branch but that isn't possible if upstream/master is
 175     # a merge commit (it will list all the commits that form part of the
 176     # merge)
 177     # Conflicting tags are not a problem for buildman, since it does not use
 178     # them. For example, Series-version is not useful for buildman. On the
 179     # other hand conflicting tags will cause an error. So allow later tags
 180     # to overwrite earlier ones by setting allow_overwrite=True
 181     if options.branch:
 182         if count == -1:
 183             if has_range:
 184                 range_expr = options.branch
 185             else:
 186                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 187                                                       options.branch)
 188             upstream_commit = gitutil.GetUpstream(options.git_dir,
 189                                                   options.branch)
 190             series = patchstream.GetMetaDataForList(upstream_commit,
 191                 options.git_dir, 1, series=None, allow_overwrite=True)
 192
 193             series = patchstream.GetMetaDataForList(range_expr,
 194                     options.git_dir, None, series, allow_overwrite=True)
 195         else:
 196             # Honour the count
 197             series = patchstream.GetMetaDataForList(options.branch,
 198                     options.git_dir, count, series=None, allow_overwrite=True)
 199     else:
 200         series = None
 201         options.verbose = True
 202         if not options.summary:
 203             options.show_errors = True
 204
 205     # By default we have one thread per CPU. But if there are not enough jobs
 206     # we can have fewer threads and use a high '-j' value for make.
 207     if not options.threads:
 208         options.threads = min(multiprocessing.cpu_count(), len(selected))
 209     if not options.jobs:
 210         options.jobs = max(1, (multiprocessing.cpu_count() +
 211                 len(selected) - 1) / len(selected))
 212
 213     if not options.step:
 214         options.step = len(series.commits) - 1
 215
 216     gnu_make = command.Output(os.path.join(options.git,
 217                                            'scripts/show-gnu-make')).rstrip()
 218     if not gnu_make:
 219         sys.exit('GNU Make not found')
 220
 221     # Create a new builder with the selected options.
 222     output_dir = options.output_dir
 223     if options.branch:
 224         dirname = options.branch.replace('/', '_')
 225         # As a special case allow the board directory to be placed in the
 226         # output directory itself rather than any subdirectory.
 227         if not options.no_subdirs:
 228             output_dir = os.path.join(options.output_dir, dirname)
 229     if (clean_dir and output_dir != options.output_dir and
 230             os.path.exists(output_dir)):
 231         shutil.rmtree(output_dir)
 232     builder = Builder(toolchains, output_dir, options.git_dir,
 233             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 234             show_unknown=options.show_unknown, step=options.step,
 235             no_subdirs=options.no_subdirs, full_path=options.full_path)
 236     builder.force_config_on_failure = not options.quick
 237     if make_func:
 238         builder.do_make = make_func
 239
 240     # For a dry run, just show our actions as a sanity check
 241     if options.dry_run:
 242         ShowActions(series, why_selected, selected, builder, options)
 243     else:
 244         builder.force_build = options.force_build
 245         builder.force_build_failures = options.force_build_failures
 246         builder.force_reconfig = options.force_reconfig
 247         builder.in_tree = options.in_tree
 248
 249         # Work out which boards to build
 250         board_selected = boards.GetSelectedDict()
 251
 252         if series:
 253             commits = series.commits
 254             # Number the commits for test purposes
 255             for commit in range(len(commits)):
 256                 commits[commit].sequence = commit
 257         else:
 258             commits = None
 259
 260         Print(GetActionSummary(options.summary, commits, board_selected,
 261                                 options))
 262
 263         # We can't show function sizes without board details at present
 264         if options.show_bloat:
 265             options.show_detail = True
 266         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 267                                   options.show_detail, options.show_bloat,
 268                                   options.list_error_boards)
 269         if options.summary:
 270             builder.ShowSummary(commits, board_selected)
 271         else:
 272             fail, warned = builder.BuildBoards(commits, board_selected,
 273                                 options.keep_outputs, options.verbose)
 274             if fail:
 275                 return 128
 276             elif warned:
 277                 return 129
 278     return 0