]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
1a459660 | 3 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
4 | # |
5 | ||
6 | import multiprocessing | |
7 | import os | |
883a321a | 8 | import shutil |
fc3fe1c2 SG |
9 | import sys |
10 | ||
11 | import board | |
12 | import bsettings | |
13 | from builder import Builder | |
14 | import gitutil | |
15 | import patchstream | |
16 | import terminal | |
d4144e45 | 17 | from terminal import Print |
fc3fe1c2 | 18 | import toolchain |
99796923 | 19 | import command |
73f30b9b | 20 | import subprocess |
fc3fe1c2 SG |
21 | |
22 | def GetPlural(count): | |
23 | """Returns a plural 's' if count is not 1""" | |
24 | return 's' if count != 1 else '' | |
25 | ||
fea5858e | 26 | def GetActionSummary(is_summary, commits, selected, options): |
fc3fe1c2 SG |
27 | """Return a string summarising the intended action. |
28 | ||
29 | Returns: | |
30 | Summary string. | |
31 | """ | |
fea5858e SG |
32 | if commits: |
33 | count = len(commits) | |
34 | count = (count + options.step - 1) / options.step | |
35 | commit_str = '%d commit%s' % (count, GetPlural(count)) | |
36 | else: | |
37 | commit_str = 'current source' | |
38 | str = '%s %s for %d boards' % ( | |
39 | 'Summary of' if is_summary else 'Building', commit_str, | |
fc3fe1c2 SG |
40 | len(selected)) |
41 | str += ' (%d thread%s, %d job%s per thread)' % (options.threads, | |
42 | GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) | |
43 | return str | |
44 | ||
45 | def ShowActions(series, why_selected, boards_selected, builder, options): | |
46 | """Display a list of actions that we would take, if not a dry run. | |
47 | ||
48 | Args: | |
49 | series: Series object | |
50 | why_selected: Dictionary where each key is a buildman argument | |
51 | provided by the user, and the value is the boards brought | |
52 | in by that argument. For example, 'arm' might bring in | |
53 | 400 boards, so in this case the key would be 'arm' and | |
54 | the value would be a list of board names. | |
55 | boards_selected: Dict of selected boards, key is target name, | |
56 | value is Board object | |
57 | builder: The builder that will be used to build the commits | |
58 | options: Command line options object | |
59 | """ | |
60 | col = terminal.Color() | |
61 | print 'Dry run, so not doing much. But I would do this:' | |
62 | ||
fea5858e SG |
63 | if series: |
64 | commits = series.commits | |
65 | else: | |
66 | commits = None | |
67 | print GetActionSummary(False, commits, boards_selected, | |
fc3fe1c2 SG |
68 | options) |
69 | print 'Build directory: %s' % builder.base_dir | |
fea5858e SG |
70 | if commits: |
71 | for upto in range(0, len(series.commits), options.step): | |
72 | commit = series.commits[upto] | |
1ddda1b3 | 73 | print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), |
fea5858e | 74 | print commit.subject |
fc3fe1c2 SG |
75 | |
76 | for arg in why_selected: | |
77 | if arg != 'all': | |
78 | print arg, ': %d boards' % why_selected[arg] | |
79 | print ('Total boards to build for each commit: %d\n' % | |
80 | why_selected['all']) | |
81 | ||
883a321a SG |
82 | def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, |
83 | clean_dir=False): | |
fc3fe1c2 SG |
84 | """The main control code for buildman |
85 | ||
86 | Args: | |
87 | options: Command line options object | |
88 | args: Command line arguments (list of strings) | |
d4144e45 SG |
89 | toolchains: Toolchains to use - this should be a Toolchains() |
90 | object. If None, then it will be created and scanned | |
91 | make_func: Make function to use for the builder. This is called | |
92 | to execute 'make'. If this is None, the normal function | |
93 | will be used, which calls the 'make' tool with suitable | |
94 | arguments. This setting is useful for tests. | |
823e60b6 SG |
95 | board: Boards() object to use, containing a list of available |
96 | boards. If this is None it will be created and scanned. | |
fc3fe1c2 | 97 | """ |
883a321a SG |
98 | global builder |
99 | ||
48ba5856 SG |
100 | if options.full_help: |
101 | pager = os.getenv('PAGER') | |
102 | if not pager: | |
103 | pager = 'more' | |
2bdeade0 SG |
104 | fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), |
105 | 'README') | |
48ba5856 SG |
106 | command.Run(pager, fname) |
107 | return 0 | |
108 | ||
fc3fe1c2 | 109 | gitutil.Setup() |
713bea38 | 110 | col = terminal.Color() |
fc3fe1c2 | 111 | |
fc3fe1c2 SG |
112 | options.git_dir = os.path.join(options.git, '.git') |
113 | ||
7e92e46e SG |
114 | no_toolchains = toolchains is None |
115 | if no_toolchains: | |
d4144e45 | 116 | toolchains = toolchain.Toolchains() |
fc3fe1c2 | 117 | |
827e37b5 SG |
118 | if options.fetch_arch: |
119 | if options.fetch_arch == 'list': | |
120 | sorted_list = toolchains.ListArchs() | |
713bea38 SG |
121 | print col.Color(col.BLUE, 'Available architectures: %s\n' % |
122 | ' '.join(sorted_list)) | |
827e37b5 SG |
123 | return 0 |
124 | else: | |
125 | fetch_arch = options.fetch_arch | |
126 | if fetch_arch == 'all': | |
127 | fetch_arch = ','.join(toolchains.ListArchs()) | |
713bea38 SG |
128 | print col.Color(col.CYAN, '\nDownloading toolchains: %s' % |
129 | fetch_arch) | |
827e37b5 | 130 | for arch in fetch_arch.split(','): |
713bea38 | 131 | |
827e37b5 SG |
132 | ret = toolchains.FetchAndInstall(arch) |
133 | if ret: | |
134 | return ret | |
135 | return 0 | |
136 | ||
7e92e46e SG |
137 | if no_toolchains: |
138 | toolchains.GetSettings() | |
139 | toolchains.Scan(options.list_tool_chains) | |
140 | if options.list_tool_chains: | |
141 | toolchains.List() | |
142 | ||
143 | return 0 | |
144 | ||
fc3fe1c2 SG |
145 | # Work out how many commits to build. We want to build everything on the |
146 | # branch. We also build the upstream commit as a control so we can see | |
147 | # problems introduced by the first commit on the branch. | |
fc3fe1c2 | 148 | count = options.count |
5abab20d | 149 | has_range = options.branch and '..' in options.branch |
fc3fe1c2 SG |
150 | if count == -1: |
151 | if not options.branch: | |
fea5858e SG |
152 | count = 1 |
153 | else: | |
5abab20d SG |
154 | if has_range: |
155 | count, msg = gitutil.CountCommitsInRange(options.git_dir, | |
156 | options.branch) | |
157 | else: | |
158 | count, msg = gitutil.CountCommitsInBranch(options.git_dir, | |
159 | options.branch) | |
fea5858e | 160 | if count is None: |
2a9e2c6a | 161 | sys.exit(col.Color(col.RED, msg)) |
5abab20d SG |
162 | elif count == 0: |
163 | sys.exit(col.Color(col.RED, "Range '%s' has no commits" % | |
164 | options.branch)) | |
2a9e2c6a SG |
165 | if msg: |
166 | print col.Color(col.YELLOW, msg) | |
fea5858e | 167 | count += 1 # Build upstream commit also |
fc3fe1c2 SG |
168 | |
169 | if not count: | |
170 | str = ("No commits found to process in branch '%s': " | |
171 | "set branch's upstream or use -c flag" % options.branch) | |
31e2141d | 172 | sys.exit(col.Color(col.RED, str)) |
fc3fe1c2 SG |
173 | |
174 | # Work out what subset of the boards we are building | |
823e60b6 SG |
175 | if not boards: |
176 | board_file = os.path.join(options.git, 'boards.cfg') | |
177 | status = subprocess.call([os.path.join(options.git, | |
178 | 'tools/genboardscfg.py')]) | |
179 | if status != 0: | |
180 | sys.exit("Failed to generate boards.cfg") | |
181 | ||
182 | boards = board.Boards() | |
183 | boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) | |
3cf4ae6f SG |
184 | |
185 | exclude = [] | |
186 | if options.exclude: | |
187 | for arg in options.exclude: | |
188 | exclude += arg.split(',') | |
189 | ||
190 | why_selected = boards.SelectBoards(args, exclude) | |
fc3fe1c2 SG |
191 | selected = boards.GetSelected() |
192 | if not len(selected): | |
31e2141d | 193 | sys.exit(col.Color(col.RED, 'No matching boards found')) |
fc3fe1c2 SG |
194 | |
195 | # Read the metadata from the commits. First look at the upstream commit, | |
196 | # then the ones in the branch. We would like to do something like | |
197 | # upstream/master~..branch but that isn't possible if upstream/master is | |
198 | # a merge commit (it will list all the commits that form part of the | |
199 | # merge) | |
950a2313 SG |
200 | # Conflicting tags are not a problem for buildman, since it does not use |
201 | # them. For example, Series-version is not useful for buildman. On the | |
202 | # other hand conflicting tags will cause an error. So allow later tags | |
203 | # to overwrite earlier ones by setting allow_overwrite=True | |
fea5858e | 204 | if options.branch: |
3b74ba5f | 205 | if count == -1: |
5abab20d SG |
206 | if has_range: |
207 | range_expr = options.branch | |
208 | else: | |
209 | range_expr = gitutil.GetRangeInBranch(options.git_dir, | |
210 | options.branch) | |
3b74ba5f SG |
211 | upstream_commit = gitutil.GetUpstream(options.git_dir, |
212 | options.branch) | |
213 | series = patchstream.GetMetaDataForList(upstream_commit, | |
950a2313 | 214 | options.git_dir, 1, series=None, allow_overwrite=True) |
3b74ba5f | 215 | |
3b74ba5f | 216 | series = patchstream.GetMetaDataForList(range_expr, |
950a2313 | 217 | options.git_dir, None, series, allow_overwrite=True) |
3b74ba5f SG |
218 | else: |
219 | # Honour the count | |
220 | series = patchstream.GetMetaDataForList(options.branch, | |
950a2313 | 221 | options.git_dir, count, series=None, allow_overwrite=True) |
fea5858e SG |
222 | else: |
223 | series = None | |
e5a0e5d8 | 224 | options.verbose = True |
58d818f1 SG |
225 | if not options.summary: |
226 | options.show_errors = True | |
fc3fe1c2 SG |
227 | |
228 | # By default we have one thread per CPU. But if there are not enough jobs | |
229 | # we can have fewer threads and use a high '-j' value for make. | |
230 | if not options.threads: | |
231 | options.threads = min(multiprocessing.cpu_count(), len(selected)) | |
232 | if not options.jobs: | |
233 | options.jobs = max(1, (multiprocessing.cpu_count() + | |
234 | len(selected) - 1) / len(selected)) | |
235 | ||
236 | if not options.step: | |
237 | options.step = len(series.commits) - 1 | |
238 | ||
99796923 MY |
239 | gnu_make = command.Output(os.path.join(options.git, |
240 | 'scripts/show-gnu-make')).rstrip() | |
241 | if not gnu_make: | |
31e2141d | 242 | sys.exit('GNU Make not found') |
99796923 | 243 | |
05c96b18 SG |
244 | # Create a new builder with the selected options. |
245 | output_dir = options.output_dir | |
fea5858e | 246 | if options.branch: |
f7582ce8 | 247 | dirname = options.branch.replace('/', '_') |
5971ab5c SG |
248 | # As a special case allow the board directory to be placed in the |
249 | # output directory itself rather than any subdirectory. | |
250 | if not options.no_subdirs: | |
251 | output_dir = os.path.join(options.output_dir, dirname) | |
0740127f SG |
252 | if (clean_dir and output_dir != options.output_dir and |
253 | os.path.exists(output_dir)): | |
883a321a | 254 | shutil.rmtree(output_dir) |
fc3fe1c2 | 255 | builder = Builder(toolchains, output_dir, options.git_dir, |
99796923 | 256 | options.threads, options.jobs, gnu_make=gnu_make, checkout=True, |
5971ab5c | 257 | show_unknown=options.show_unknown, step=options.step, |
d2ce658d | 258 | no_subdirs=options.no_subdirs, full_path=options.full_path, |
f79f1e0c SW |
259 | verbose_build=options.verbose_build, |
260 | incremental=options.incremental, | |
261 | per_board_out_dir=options.per_board_out_dir,) | |
fc3fe1c2 | 262 | builder.force_config_on_failure = not options.quick |
d4144e45 SG |
263 | if make_func: |
264 | builder.do_make = make_func | |
fc3fe1c2 SG |
265 | |
266 | # For a dry run, just show our actions as a sanity check | |
267 | if options.dry_run: | |
268 | ShowActions(series, why_selected, selected, builder, options) | |
269 | else: | |
270 | builder.force_build = options.force_build | |
4266dc28 | 271 | builder.force_build_failures = options.force_build_failures |
97e91526 | 272 | builder.force_reconfig = options.force_reconfig |
189a4968 | 273 | builder.in_tree = options.in_tree |
fc3fe1c2 SG |
274 | |
275 | # Work out which boards to build | |
276 | board_selected = boards.GetSelectedDict() | |
277 | ||
fea5858e SG |
278 | if series: |
279 | commits = series.commits | |
883a321a SG |
280 | # Number the commits for test purposes |
281 | for commit in range(len(commits)): | |
282 | commits[commit].sequence = commit | |
fea5858e SG |
283 | else: |
284 | commits = None | |
285 | ||
d4144e45 SG |
286 | Print(GetActionSummary(options.summary, commits, board_selected, |
287 | options)) | |
fc3fe1c2 | 288 | |
7798e228 SG |
289 | # We can't show function sizes without board details at present |
290 | if options.show_bloat: | |
291 | options.show_detail = True | |
b2ea7ab2 | 292 | builder.SetDisplayOptions(options.show_errors, options.show_sizes, |
ed966657 | 293 | options.show_detail, options.show_bloat, |
843312dc SG |
294 | options.list_error_boards, |
295 | options.show_config) | |
fc3fe1c2 | 296 | if options.summary: |
b2ea7ab2 | 297 | builder.ShowSummary(commits, board_selected) |
fc3fe1c2 | 298 | else: |
2c3deb97 | 299 | fail, warned = builder.BuildBoards(commits, board_selected, |
e5a0e5d8 | 300 | options.keep_outputs, options.verbose) |
2c3deb97 SG |
301 | if fail: |
302 | return 128 | |
303 | elif warned: | |
304 | return 129 | |
305 | return 0 |