]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
1a459660 | 3 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
4 | # |
5 | ||
6 | import multiprocessing | |
7 | import os | |
883a321a | 8 | import shutil |
fc3fe1c2 SG |
9 | import sys |
10 | ||
11 | import board | |
12 | import bsettings | |
13 | from builder import Builder | |
14 | import gitutil | |
15 | import patchstream | |
16 | import terminal | |
d4144e45 | 17 | from terminal import Print |
fc3fe1c2 | 18 | import toolchain |
99796923 | 19 | import command |
73f30b9b | 20 | import subprocess |
fc3fe1c2 SG |
21 | |
22 | def GetPlural(count): | |
23 | """Returns a plural 's' if count is not 1""" | |
24 | return 's' if count != 1 else '' | |
25 | ||
fea5858e | 26 | def GetActionSummary(is_summary, commits, selected, options): |
fc3fe1c2 SG |
27 | """Return a string summarising the intended action. |
28 | ||
29 | Returns: | |
30 | Summary string. | |
31 | """ | |
fea5858e SG |
32 | if commits: |
33 | count = len(commits) | |
34 | count = (count + options.step - 1) / options.step | |
35 | commit_str = '%d commit%s' % (count, GetPlural(count)) | |
36 | else: | |
37 | commit_str = 'current source' | |
38 | str = '%s %s for %d boards' % ( | |
39 | 'Summary of' if is_summary else 'Building', commit_str, | |
fc3fe1c2 SG |
40 | len(selected)) |
41 | str += ' (%d thread%s, %d job%s per thread)' % (options.threads, | |
42 | GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) | |
43 | return str | |
44 | ||
45 | def ShowActions(series, why_selected, boards_selected, builder, options): | |
46 | """Display a list of actions that we would take, if not a dry run. | |
47 | ||
48 | Args: | |
49 | series: Series object | |
50 | why_selected: Dictionary where each key is a buildman argument | |
51 | provided by the user, and the value is the boards brought | |
52 | in by that argument. For example, 'arm' might bring in | |
53 | 400 boards, so in this case the key would be 'arm' and | |
54 | the value would be a list of board names. | |
55 | boards_selected: Dict of selected boards, key is target name, | |
56 | value is Board object | |
57 | builder: The builder that will be used to build the commits | |
58 | options: Command line options object | |
59 | """ | |
60 | col = terminal.Color() | |
61 | print 'Dry run, so not doing much. But I would do this:' | |
62 | ||
fea5858e SG |
63 | if series: |
64 | commits = series.commits | |
65 | else: | |
66 | commits = None | |
67 | print GetActionSummary(False, commits, boards_selected, | |
fc3fe1c2 SG |
68 | options) |
69 | print 'Build directory: %s' % builder.base_dir | |
fea5858e SG |
70 | if commits: |
71 | for upto in range(0, len(series.commits), options.step): | |
72 | commit = series.commits[upto] | |
1ddda1b3 | 73 | print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), |
fea5858e | 74 | print commit.subject |
fc3fe1c2 SG |
75 | |
76 | for arg in why_selected: | |
77 | if arg != 'all': | |
78 | print arg, ': %d boards' % why_selected[arg] | |
79 | print ('Total boards to build for each commit: %d\n' % | |
80 | why_selected['all']) | |
81 | ||
883a321a SG |
82 | def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, |
83 | clean_dir=False): | |
fc3fe1c2 SG |
84 | """The main control code for buildman |
85 | ||
86 | Args: | |
87 | options: Command line options object | |
88 | args: Command line arguments (list of strings) | |
d4144e45 SG |
89 | toolchains: Toolchains to use - this should be a Toolchains() |
90 | object. If None, then it will be created and scanned | |
91 | make_func: Make function to use for the builder. This is called | |
92 | to execute 'make'. If this is None, the normal function | |
93 | will be used, which calls the 'make' tool with suitable | |
94 | arguments. This setting is useful for tests. | |
823e60b6 SG |
95 | board: Boards() object to use, containing a list of available |
96 | boards. If this is None it will be created and scanned. | |
fc3fe1c2 | 97 | """ |
883a321a SG |
98 | global builder |
99 | ||
48ba5856 SG |
100 | if options.full_help: |
101 | pager = os.getenv('PAGER') | |
102 | if not pager: | |
103 | pager = 'more' | |
104 | fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') | |
105 | command.Run(pager, fname) | |
106 | return 0 | |
107 | ||
fc3fe1c2 SG |
108 | gitutil.Setup() |
109 | ||
fc3fe1c2 SG |
110 | options.git_dir = os.path.join(options.git, '.git') |
111 | ||
d4144e45 SG |
112 | if not toolchains: |
113 | toolchains = toolchain.Toolchains() | |
114 | toolchains.GetSettings() | |
115 | toolchains.Scan(options.list_tool_chains) | |
fc3fe1c2 SG |
116 | if options.list_tool_chains: |
117 | toolchains.List() | |
118 | ||
2c3deb97 | 119 | return 0 |
fc3fe1c2 SG |
120 | |
121 | # Work out how many commits to build. We want to build everything on the | |
122 | # branch. We also build the upstream commit as a control so we can see | |
123 | # problems introduced by the first commit on the branch. | |
124 | col = terminal.Color() | |
125 | count = options.count | |
5abab20d | 126 | has_range = options.branch and '..' in options.branch |
fc3fe1c2 SG |
127 | if count == -1: |
128 | if not options.branch: | |
fea5858e SG |
129 | count = 1 |
130 | else: | |
5abab20d SG |
131 | if has_range: |
132 | count, msg = gitutil.CountCommitsInRange(options.git_dir, | |
133 | options.branch) | |
134 | else: | |
135 | count, msg = gitutil.CountCommitsInBranch(options.git_dir, | |
136 | options.branch) | |
fea5858e | 137 | if count is None: |
2a9e2c6a | 138 | sys.exit(col.Color(col.RED, msg)) |
5abab20d SG |
139 | elif count == 0: |
140 | sys.exit(col.Color(col.RED, "Range '%s' has no commits" % | |
141 | options.branch)) | |
2a9e2c6a SG |
142 | if msg: |
143 | print col.Color(col.YELLOW, msg) | |
fea5858e | 144 | count += 1 # Build upstream commit also |
fc3fe1c2 SG |
145 | |
146 | if not count: | |
147 | str = ("No commits found to process in branch '%s': " | |
148 | "set branch's upstream or use -c flag" % options.branch) | |
31e2141d | 149 | sys.exit(col.Color(col.RED, str)) |
fc3fe1c2 SG |
150 | |
151 | # Work out what subset of the boards we are building | |
823e60b6 SG |
152 | if not boards: |
153 | board_file = os.path.join(options.git, 'boards.cfg') | |
154 | status = subprocess.call([os.path.join(options.git, | |
155 | 'tools/genboardscfg.py')]) | |
156 | if status != 0: | |
157 | sys.exit("Failed to generate boards.cfg") | |
158 | ||
159 | boards = board.Boards() | |
160 | boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) | |
3cf4ae6f SG |
161 | |
162 | exclude = [] | |
163 | if options.exclude: | |
164 | for arg in options.exclude: | |
165 | exclude += arg.split(',') | |
166 | ||
167 | why_selected = boards.SelectBoards(args, exclude) | |
fc3fe1c2 SG |
168 | selected = boards.GetSelected() |
169 | if not len(selected): | |
31e2141d | 170 | sys.exit(col.Color(col.RED, 'No matching boards found')) |
fc3fe1c2 SG |
171 | |
172 | # Read the metadata from the commits. First look at the upstream commit, | |
173 | # then the ones in the branch. We would like to do something like | |
174 | # upstream/master~..branch but that isn't possible if upstream/master is | |
175 | # a merge commit (it will list all the commits that form part of the | |
176 | # merge) | |
950a2313 SG |
177 | # Conflicting tags are not a problem for buildman, since it does not use |
178 | # them. For example, Series-version is not useful for buildman. On the | |
179 | # other hand conflicting tags will cause an error. So allow later tags | |
180 | # to overwrite earlier ones by setting allow_overwrite=True | |
fea5858e | 181 | if options.branch: |
3b74ba5f | 182 | if count == -1: |
5abab20d SG |
183 | if has_range: |
184 | range_expr = options.branch | |
185 | else: | |
186 | range_expr = gitutil.GetRangeInBranch(options.git_dir, | |
187 | options.branch) | |
3b74ba5f SG |
188 | upstream_commit = gitutil.GetUpstream(options.git_dir, |
189 | options.branch) | |
190 | series = patchstream.GetMetaDataForList(upstream_commit, | |
950a2313 | 191 | options.git_dir, 1, series=None, allow_overwrite=True) |
3b74ba5f | 192 | |
3b74ba5f | 193 | series = patchstream.GetMetaDataForList(range_expr, |
950a2313 | 194 | options.git_dir, None, series, allow_overwrite=True) |
3b74ba5f SG |
195 | else: |
196 | # Honour the count | |
197 | series = patchstream.GetMetaDataForList(options.branch, | |
950a2313 | 198 | options.git_dir, count, series=None, allow_overwrite=True) |
fea5858e SG |
199 | else: |
200 | series = None | |
e5a0e5d8 | 201 | options.verbose = True |
58d818f1 SG |
202 | if not options.summary: |
203 | options.show_errors = True | |
fc3fe1c2 SG |
204 | |
205 | # By default we have one thread per CPU. But if there are not enough jobs | |
206 | # we can have fewer threads and use a high '-j' value for make. | |
207 | if not options.threads: | |
208 | options.threads = min(multiprocessing.cpu_count(), len(selected)) | |
209 | if not options.jobs: | |
210 | options.jobs = max(1, (multiprocessing.cpu_count() + | |
211 | len(selected) - 1) / len(selected)) | |
212 | ||
213 | if not options.step: | |
214 | options.step = len(series.commits) - 1 | |
215 | ||
99796923 MY |
216 | gnu_make = command.Output(os.path.join(options.git, |
217 | 'scripts/show-gnu-make')).rstrip() | |
218 | if not gnu_make: | |
31e2141d | 219 | sys.exit('GNU Make not found') |
99796923 | 220 | |
05c96b18 SG |
221 | # Create a new builder with the selected options. |
222 | output_dir = options.output_dir | |
fea5858e | 223 | if options.branch: |
f7582ce8 | 224 | dirname = options.branch.replace('/', '_') |
5971ab5c SG |
225 | # As a special case allow the board directory to be placed in the |
226 | # output directory itself rather than any subdirectory. | |
227 | if not options.no_subdirs: | |
228 | output_dir = os.path.join(options.output_dir, dirname) | |
0740127f SG |
229 | if (clean_dir and output_dir != options.output_dir and |
230 | os.path.exists(output_dir)): | |
883a321a | 231 | shutil.rmtree(output_dir) |
fc3fe1c2 | 232 | builder = Builder(toolchains, output_dir, options.git_dir, |
99796923 | 233 | options.threads, options.jobs, gnu_make=gnu_make, checkout=True, |
5971ab5c | 234 | show_unknown=options.show_unknown, step=options.step, |
bb1501f2 | 235 | no_subdirs=options.no_subdirs, full_path=options.full_path) |
fc3fe1c2 | 236 | builder.force_config_on_failure = not options.quick |
d4144e45 SG |
237 | if make_func: |
238 | builder.do_make = make_func | |
fc3fe1c2 SG |
239 | |
240 | # For a dry run, just show our actions as a sanity check | |
241 | if options.dry_run: | |
242 | ShowActions(series, why_selected, selected, builder, options) | |
243 | else: | |
244 | builder.force_build = options.force_build | |
4266dc28 | 245 | builder.force_build_failures = options.force_build_failures |
97e91526 | 246 | builder.force_reconfig = options.force_reconfig |
189a4968 | 247 | builder.in_tree = options.in_tree |
fc3fe1c2 SG |
248 | |
249 | # Work out which boards to build | |
250 | board_selected = boards.GetSelectedDict() | |
251 | ||
fea5858e SG |
252 | if series: |
253 | commits = series.commits | |
883a321a SG |
254 | # Number the commits for test purposes |
255 | for commit in range(len(commits)): | |
256 | commits[commit].sequence = commit | |
fea5858e SG |
257 | else: |
258 | commits = None | |
259 | ||
d4144e45 SG |
260 | Print(GetActionSummary(options.summary, commits, board_selected, |
261 | options)) | |
fc3fe1c2 | 262 | |
7798e228 SG |
263 | # We can't show function sizes without board details at present |
264 | if options.show_bloat: | |
265 | options.show_detail = True | |
b2ea7ab2 | 266 | builder.SetDisplayOptions(options.show_errors, options.show_sizes, |
ed966657 SG |
267 | options.show_detail, options.show_bloat, |
268 | options.list_error_boards) | |
fc3fe1c2 | 269 | if options.summary: |
b2ea7ab2 | 270 | builder.ShowSummary(commits, board_selected) |
fc3fe1c2 | 271 | else: |
2c3deb97 | 272 | fail, warned = builder.BuildBoards(commits, board_selected, |
e5a0e5d8 | 273 | options.keep_outputs, options.verbose) |
2c3deb97 SG |
274 | if fail: |
275 | return 128 | |
276 | elif warned: | |
277 | return 129 | |
278 | return 0 |