]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
1a459660 | 3 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
4 | # |
5 | ||
6 | import multiprocessing | |
7 | import os | |
883a321a | 8 | import shutil |
fc3fe1c2 SG |
9 | import sys |
10 | ||
11 | import board | |
12 | import bsettings | |
13 | from builder import Builder | |
14 | import gitutil | |
15 | import patchstream | |
16 | import terminal | |
d4144e45 | 17 | from terminal import Print |
fc3fe1c2 | 18 | import toolchain |
99796923 | 19 | import command |
73f30b9b | 20 | import subprocess |
fc3fe1c2 SG |
21 | |
22 | def GetPlural(count): | |
23 | """Returns a plural 's' if count is not 1""" | |
24 | return 's' if count != 1 else '' | |
25 | ||
fea5858e | 26 | def GetActionSummary(is_summary, commits, selected, options): |
fc3fe1c2 SG |
27 | """Return a string summarising the intended action. |
28 | ||
29 | Returns: | |
30 | Summary string. | |
31 | """ | |
fea5858e SG |
32 | if commits: |
33 | count = len(commits) | |
34 | count = (count + options.step - 1) / options.step | |
35 | commit_str = '%d commit%s' % (count, GetPlural(count)) | |
36 | else: | |
37 | commit_str = 'current source' | |
38 | str = '%s %s for %d boards' % ( | |
39 | 'Summary of' if is_summary else 'Building', commit_str, | |
fc3fe1c2 SG |
40 | len(selected)) |
41 | str += ' (%d thread%s, %d job%s per thread)' % (options.threads, | |
42 | GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) | |
43 | return str | |
44 | ||
45 | def ShowActions(series, why_selected, boards_selected, builder, options): | |
46 | """Display a list of actions that we would take, if not a dry run. | |
47 | ||
48 | Args: | |
49 | series: Series object | |
50 | why_selected: Dictionary where each key is a buildman argument | |
8d7523c5 SG |
51 | provided by the user, and the value is the list of boards |
52 | brought in by that argument. For example, 'arm' might bring | |
53 | in 400 boards, so in this case the key would be 'arm' and | |
fc3fe1c2 SG |
54 | the value would be a list of board names. |
55 | boards_selected: Dict of selected boards, key is target name, | |
56 | value is Board object | |
57 | builder: The builder that will be used to build the commits | |
58 | options: Command line options object | |
59 | """ | |
60 | col = terminal.Color() | |
61 | print 'Dry run, so not doing much. But I would do this:' | |
62 | ||
fea5858e SG |
63 | if series: |
64 | commits = series.commits | |
65 | else: | |
66 | commits = None | |
67 | print GetActionSummary(False, commits, boards_selected, | |
fc3fe1c2 SG |
68 | options) |
69 | print 'Build directory: %s' % builder.base_dir | |
fea5858e SG |
70 | if commits: |
71 | for upto in range(0, len(series.commits), options.step): | |
72 | commit = series.commits[upto] | |
1ddda1b3 | 73 | print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), |
fea5858e | 74 | print commit.subject |
fc3fe1c2 SG |
75 | |
76 | for arg in why_selected: | |
77 | if arg != 'all': | |
8d7523c5 SG |
78 | print arg, ': %d boards' % len(why_selected[arg]) |
79 | if options.verbose: | |
80 | print ' %s' % ' '.join(why_selected[arg]) | |
fc3fe1c2 | 81 | print ('Total boards to build for each commit: %d\n' % |
8d7523c5 | 82 | len(why_selected['all'])) |
fc3fe1c2 | 83 | |
883a321a SG |
84 | def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, |
85 | clean_dir=False): | |
fc3fe1c2 SG |
86 | """The main control code for buildman |
87 | ||
88 | Args: | |
89 | options: Command line options object | |
90 | args: Command line arguments (list of strings) | |
d4144e45 SG |
91 | toolchains: Toolchains to use - this should be a Toolchains() |
92 | object. If None, then it will be created and scanned | |
93 | make_func: Make function to use for the builder. This is called | |
94 | to execute 'make'. If this is None, the normal function | |
95 | will be used, which calls the 'make' tool with suitable | |
96 | arguments. This setting is useful for tests. | |
823e60b6 SG |
97 | board: Boards() object to use, containing a list of available |
98 | boards. If this is None it will be created and scanned. | |
fc3fe1c2 | 99 | """ |
883a321a SG |
100 | global builder |
101 | ||
48ba5856 SG |
102 | if options.full_help: |
103 | pager = os.getenv('PAGER') | |
104 | if not pager: | |
105 | pager = 'more' | |
2bdeade0 SG |
106 | fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), |
107 | 'README') | |
48ba5856 SG |
108 | command.Run(pager, fname) |
109 | return 0 | |
110 | ||
fc3fe1c2 | 111 | gitutil.Setup() |
713bea38 | 112 | col = terminal.Color() |
fc3fe1c2 | 113 | |
fc3fe1c2 SG |
114 | options.git_dir = os.path.join(options.git, '.git') |
115 | ||
7e92e46e SG |
116 | no_toolchains = toolchains is None |
117 | if no_toolchains: | |
d4144e45 | 118 | toolchains = toolchain.Toolchains() |
fc3fe1c2 | 119 | |
827e37b5 SG |
120 | if options.fetch_arch: |
121 | if options.fetch_arch == 'list': | |
122 | sorted_list = toolchains.ListArchs() | |
713bea38 SG |
123 | print col.Color(col.BLUE, 'Available architectures: %s\n' % |
124 | ' '.join(sorted_list)) | |
827e37b5 SG |
125 | return 0 |
126 | else: | |
127 | fetch_arch = options.fetch_arch | |
128 | if fetch_arch == 'all': | |
129 | fetch_arch = ','.join(toolchains.ListArchs()) | |
713bea38 SG |
130 | print col.Color(col.CYAN, '\nDownloading toolchains: %s' % |
131 | fetch_arch) | |
827e37b5 | 132 | for arch in fetch_arch.split(','): |
713bea38 | 133 | |
827e37b5 SG |
134 | ret = toolchains.FetchAndInstall(arch) |
135 | if ret: | |
136 | return ret | |
137 | return 0 | |
138 | ||
7e92e46e SG |
139 | if no_toolchains: |
140 | toolchains.GetSettings() | |
141 | toolchains.Scan(options.list_tool_chains) | |
142 | if options.list_tool_chains: | |
143 | toolchains.List() | |
144 | ||
145 | return 0 | |
146 | ||
fc3fe1c2 SG |
147 | # Work out how many commits to build. We want to build everything on the |
148 | # branch. We also build the upstream commit as a control so we can see | |
149 | # problems introduced by the first commit on the branch. | |
fc3fe1c2 | 150 | count = options.count |
5abab20d | 151 | has_range = options.branch and '..' in options.branch |
fc3fe1c2 SG |
152 | if count == -1: |
153 | if not options.branch: | |
fea5858e SG |
154 | count = 1 |
155 | else: | |
5abab20d SG |
156 | if has_range: |
157 | count, msg = gitutil.CountCommitsInRange(options.git_dir, | |
158 | options.branch) | |
159 | else: | |
160 | count, msg = gitutil.CountCommitsInBranch(options.git_dir, | |
161 | options.branch) | |
fea5858e | 162 | if count is None: |
2a9e2c6a | 163 | sys.exit(col.Color(col.RED, msg)) |
5abab20d SG |
164 | elif count == 0: |
165 | sys.exit(col.Color(col.RED, "Range '%s' has no commits" % | |
166 | options.branch)) | |
2a9e2c6a SG |
167 | if msg: |
168 | print col.Color(col.YELLOW, msg) | |
fea5858e | 169 | count += 1 # Build upstream commit also |
fc3fe1c2 SG |
170 | |
171 | if not count: | |
172 | str = ("No commits found to process in branch '%s': " | |
173 | "set branch's upstream or use -c flag" % options.branch) | |
31e2141d | 174 | sys.exit(col.Color(col.RED, str)) |
fc3fe1c2 SG |
175 | |
176 | # Work out what subset of the boards we are building | |
823e60b6 SG |
177 | if not boards: |
178 | board_file = os.path.join(options.git, 'boards.cfg') | |
179 | status = subprocess.call([os.path.join(options.git, | |
180 | 'tools/genboardscfg.py')]) | |
181 | if status != 0: | |
182 | sys.exit("Failed to generate boards.cfg") | |
183 | ||
184 | boards = board.Boards() | |
185 | boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) | |
3cf4ae6f SG |
186 | |
187 | exclude = [] | |
188 | if options.exclude: | |
189 | for arg in options.exclude: | |
190 | exclude += arg.split(',') | |
191 | ||
192 | why_selected = boards.SelectBoards(args, exclude) | |
fc3fe1c2 SG |
193 | selected = boards.GetSelected() |
194 | if not len(selected): | |
31e2141d | 195 | sys.exit(col.Color(col.RED, 'No matching boards found')) |
fc3fe1c2 SG |
196 | |
197 | # Read the metadata from the commits. First look at the upstream commit, | |
198 | # then the ones in the branch. We would like to do something like | |
199 | # upstream/master~..branch but that isn't possible if upstream/master is | |
200 | # a merge commit (it will list all the commits that form part of the | |
201 | # merge) | |
950a2313 SG |
202 | # Conflicting tags are not a problem for buildman, since it does not use |
203 | # them. For example, Series-version is not useful for buildman. On the | |
204 | # other hand conflicting tags will cause an error. So allow later tags | |
205 | # to overwrite earlier ones by setting allow_overwrite=True | |
fea5858e | 206 | if options.branch: |
3b74ba5f | 207 | if count == -1: |
5abab20d SG |
208 | if has_range: |
209 | range_expr = options.branch | |
210 | else: | |
211 | range_expr = gitutil.GetRangeInBranch(options.git_dir, | |
212 | options.branch) | |
3b74ba5f SG |
213 | upstream_commit = gitutil.GetUpstream(options.git_dir, |
214 | options.branch) | |
215 | series = patchstream.GetMetaDataForList(upstream_commit, | |
950a2313 | 216 | options.git_dir, 1, series=None, allow_overwrite=True) |
3b74ba5f | 217 | |
3b74ba5f | 218 | series = patchstream.GetMetaDataForList(range_expr, |
950a2313 | 219 | options.git_dir, None, series, allow_overwrite=True) |
3b74ba5f SG |
220 | else: |
221 | # Honour the count | |
222 | series = patchstream.GetMetaDataForList(options.branch, | |
950a2313 | 223 | options.git_dir, count, series=None, allow_overwrite=True) |
fea5858e SG |
224 | else: |
225 | series = None | |
8d7523c5 SG |
226 | if not options.dry_run: |
227 | options.verbose = True | |
228 | if not options.summary: | |
229 | options.show_errors = True | |
fc3fe1c2 SG |
230 | |
231 | # By default we have one thread per CPU. But if there are not enough jobs | |
232 | # we can have fewer threads and use a high '-j' value for make. | |
233 | if not options.threads: | |
234 | options.threads = min(multiprocessing.cpu_count(), len(selected)) | |
235 | if not options.jobs: | |
236 | options.jobs = max(1, (multiprocessing.cpu_count() + | |
237 | len(selected) - 1) / len(selected)) | |
238 | ||
239 | if not options.step: | |
240 | options.step = len(series.commits) - 1 | |
241 | ||
99796923 | 242 | gnu_make = command.Output(os.path.join(options.git, |
785f1548 | 243 | 'scripts/show-gnu-make'), raise_on_error=False).rstrip() |
99796923 | 244 | if not gnu_make: |
31e2141d | 245 | sys.exit('GNU Make not found') |
99796923 | 246 | |
05c96b18 SG |
247 | # Create a new builder with the selected options. |
248 | output_dir = options.output_dir | |
fea5858e | 249 | if options.branch: |
f7582ce8 | 250 | dirname = options.branch.replace('/', '_') |
5971ab5c SG |
251 | # As a special case allow the board directory to be placed in the |
252 | # output directory itself rather than any subdirectory. | |
253 | if not options.no_subdirs: | |
254 | output_dir = os.path.join(options.output_dir, dirname) | |
0740127f SG |
255 | if (clean_dir and output_dir != options.output_dir and |
256 | os.path.exists(output_dir)): | |
883a321a | 257 | shutil.rmtree(output_dir) |
fc3fe1c2 | 258 | builder = Builder(toolchains, output_dir, options.git_dir, |
99796923 | 259 | options.threads, options.jobs, gnu_make=gnu_make, checkout=True, |
5971ab5c | 260 | show_unknown=options.show_unknown, step=options.step, |
d2ce658d | 261 | no_subdirs=options.no_subdirs, full_path=options.full_path, |
f79f1e0c SW |
262 | verbose_build=options.verbose_build, |
263 | incremental=options.incremental, | |
b50113f3 | 264 | per_board_out_dir=options.per_board_out_dir, |
b464f8e7 | 265 | config_only=options.config_only, |
2371d1bc DS |
266 | squash_config_y=not options.preserve_config_y, |
267 | warnings_as_errors=options.warnings_as_errors) | |
fc3fe1c2 | 268 | builder.force_config_on_failure = not options.quick |
d4144e45 SG |
269 | if make_func: |
270 | builder.do_make = make_func | |
fc3fe1c2 SG |
271 | |
272 | # For a dry run, just show our actions as a sanity check | |
273 | if options.dry_run: | |
274 | ShowActions(series, why_selected, selected, builder, options) | |
275 | else: | |
276 | builder.force_build = options.force_build | |
4266dc28 | 277 | builder.force_build_failures = options.force_build_failures |
97e91526 | 278 | builder.force_reconfig = options.force_reconfig |
189a4968 | 279 | builder.in_tree = options.in_tree |
fc3fe1c2 SG |
280 | |
281 | # Work out which boards to build | |
282 | board_selected = boards.GetSelectedDict() | |
283 | ||
fea5858e SG |
284 | if series: |
285 | commits = series.commits | |
883a321a SG |
286 | # Number the commits for test purposes |
287 | for commit in range(len(commits)): | |
288 | commits[commit].sequence = commit | |
fea5858e SG |
289 | else: |
290 | commits = None | |
291 | ||
d4144e45 SG |
292 | Print(GetActionSummary(options.summary, commits, board_selected, |
293 | options)) | |
fc3fe1c2 | 294 | |
7798e228 SG |
295 | # We can't show function sizes without board details at present |
296 | if options.show_bloat: | |
297 | options.show_detail = True | |
b2ea7ab2 | 298 | builder.SetDisplayOptions(options.show_errors, options.show_sizes, |
ed966657 | 299 | options.show_detail, options.show_bloat, |
843312dc SG |
300 | options.list_error_boards, |
301 | options.show_config) | |
fc3fe1c2 | 302 | if options.summary: |
b2ea7ab2 | 303 | builder.ShowSummary(commits, board_selected) |
fc3fe1c2 | 304 | else: |
2c3deb97 | 305 | fail, warned = builder.BuildBoards(commits, board_selected, |
e5a0e5d8 | 306 | options.keep_outputs, options.verbose) |
2c3deb97 SG |
307 | if fail: |
308 | return 128 | |
309 | elif warned: | |
310 | return 129 | |
311 | return 0 |