]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
1a459660 | 3 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
4 | # |
5 | ||
6 | import multiprocessing | |
7 | import os | |
883a321a | 8 | import shutil |
fc3fe1c2 SG |
9 | import sys |
10 | ||
11 | import board | |
12 | import bsettings | |
13 | from builder import Builder | |
14 | import gitutil | |
15 | import patchstream | |
16 | import terminal | |
d4144e45 | 17 | from terminal import Print |
fc3fe1c2 | 18 | import toolchain |
99796923 | 19 | import command |
73f30b9b | 20 | import subprocess |
fc3fe1c2 SG |
21 | |
22 | def GetPlural(count): | |
23 | """Returns a plural 's' if count is not 1""" | |
24 | return 's' if count != 1 else '' | |
25 | ||
fea5858e | 26 | def GetActionSummary(is_summary, commits, selected, options): |
fc3fe1c2 SG |
27 | """Return a string summarising the intended action. |
28 | ||
29 | Returns: | |
30 | Summary string. | |
31 | """ | |
fea5858e SG |
32 | if commits: |
33 | count = len(commits) | |
34 | count = (count + options.step - 1) / options.step | |
35 | commit_str = '%d commit%s' % (count, GetPlural(count)) | |
36 | else: | |
37 | commit_str = 'current source' | |
38 | str = '%s %s for %d boards' % ( | |
39 | 'Summary of' if is_summary else 'Building', commit_str, | |
fc3fe1c2 SG |
40 | len(selected)) |
41 | str += ' (%d thread%s, %d job%s per thread)' % (options.threads, | |
42 | GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) | |
43 | return str | |
44 | ||
45 | def ShowActions(series, why_selected, boards_selected, builder, options): | |
46 | """Display a list of actions that we would take, if not a dry run. | |
47 | ||
48 | Args: | |
49 | series: Series object | |
50 | why_selected: Dictionary where each key is a buildman argument | |
51 | provided by the user, and the value is the boards brought | |
52 | in by that argument. For example, 'arm' might bring in | |
53 | 400 boards, so in this case the key would be 'arm' and | |
54 | the value would be a list of board names. | |
55 | boards_selected: Dict of selected boards, key is target name, | |
56 | value is Board object | |
57 | builder: The builder that will be used to build the commits | |
58 | options: Command line options object | |
59 | """ | |
60 | col = terminal.Color() | |
61 | print 'Dry run, so not doing much. But I would do this:' | |
62 | ||
fea5858e SG |
63 | if series: |
64 | commits = series.commits | |
65 | else: | |
66 | commits = None | |
67 | print GetActionSummary(False, commits, boards_selected, | |
fc3fe1c2 SG |
68 | options) |
69 | print 'Build directory: %s' % builder.base_dir | |
fea5858e SG |
70 | if commits: |
71 | for upto in range(0, len(series.commits), options.step): | |
72 | commit = series.commits[upto] | |
73 | print ' ', col.Color(col.YELLOW, commit.hash, bright=False), | |
74 | print commit.subject | |
fc3fe1c2 SG |
75 | |
76 | for arg in why_selected: | |
77 | if arg != 'all': | |
78 | print arg, ': %d boards' % why_selected[arg] | |
79 | print ('Total boards to build for each commit: %d\n' % | |
80 | why_selected['all']) | |
81 | ||
883a321a SG |
82 | def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, |
83 | clean_dir=False): | |
fc3fe1c2 SG |
84 | """The main control code for buildman |
85 | ||
86 | Args: | |
87 | options: Command line options object | |
88 | args: Command line arguments (list of strings) | |
d4144e45 SG |
89 | toolchains: Toolchains to use - this should be a Toolchains() |
90 | object. If None, then it will be created and scanned | |
91 | make_func: Make function to use for the builder. This is called | |
92 | to execute 'make'. If this is None, the normal function | |
93 | will be used, which calls the 'make' tool with suitable | |
94 | arguments. This setting is useful for tests. | |
823e60b6 SG |
95 | board: Boards() object to use, containing a list of available |
96 | boards. If this is None it will be created and scanned. | |
fc3fe1c2 | 97 | """ |
883a321a SG |
98 | global builder |
99 | ||
48ba5856 SG |
100 | if options.full_help: |
101 | pager = os.getenv('PAGER') | |
102 | if not pager: | |
103 | pager = 'more' | |
104 | fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') | |
105 | command.Run(pager, fname) | |
106 | return 0 | |
107 | ||
fc3fe1c2 SG |
108 | gitutil.Setup() |
109 | ||
fc3fe1c2 SG |
110 | options.git_dir = os.path.join(options.git, '.git') |
111 | ||
d4144e45 SG |
112 | if not toolchains: |
113 | toolchains = toolchain.Toolchains() | |
114 | toolchains.GetSettings() | |
115 | toolchains.Scan(options.list_tool_chains) | |
fc3fe1c2 SG |
116 | if options.list_tool_chains: |
117 | toolchains.List() | |
118 | ||
2c3deb97 | 119 | return 0 |
fc3fe1c2 SG |
120 | |
121 | # Work out how many commits to build. We want to build everything on the | |
122 | # branch. We also build the upstream commit as a control so we can see | |
123 | # problems introduced by the first commit on the branch. | |
124 | col = terminal.Color() | |
125 | count = options.count | |
126 | if count == -1: | |
127 | if not options.branch: | |
fea5858e SG |
128 | count = 1 |
129 | else: | |
130 | count = gitutil.CountCommitsInBranch(options.git_dir, | |
131 | options.branch) | |
132 | if count is None: | |
133 | str = ("Branch '%s' not found or has no upstream" % | |
134 | options.branch) | |
31e2141d | 135 | sys.exit(col.Color(col.RED, str)) |
fea5858e | 136 | count += 1 # Build upstream commit also |
fc3fe1c2 SG |
137 | |
138 | if not count: | |
139 | str = ("No commits found to process in branch '%s': " | |
140 | "set branch's upstream or use -c flag" % options.branch) | |
31e2141d | 141 | sys.exit(col.Color(col.RED, str)) |
fc3fe1c2 SG |
142 | |
143 | # Work out what subset of the boards we are building | |
823e60b6 SG |
144 | if not boards: |
145 | board_file = os.path.join(options.git, 'boards.cfg') | |
146 | status = subprocess.call([os.path.join(options.git, | |
147 | 'tools/genboardscfg.py')]) | |
148 | if status != 0: | |
149 | sys.exit("Failed to generate boards.cfg") | |
150 | ||
151 | boards = board.Boards() | |
152 | boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) | |
3cf4ae6f SG |
153 | |
154 | exclude = [] | |
155 | if options.exclude: | |
156 | for arg in options.exclude: | |
157 | exclude += arg.split(',') | |
158 | ||
159 | why_selected = boards.SelectBoards(args, exclude) | |
fc3fe1c2 SG |
160 | selected = boards.GetSelected() |
161 | if not len(selected): | |
31e2141d | 162 | sys.exit(col.Color(col.RED, 'No matching boards found')) |
fc3fe1c2 SG |
163 | |
164 | # Read the metadata from the commits. First look at the upstream commit, | |
165 | # then the ones in the branch. We would like to do something like | |
166 | # upstream/master~..branch but that isn't possible if upstream/master is | |
167 | # a merge commit (it will list all the commits that form part of the | |
168 | # merge) | |
950a2313 SG |
169 | # Conflicting tags are not a problem for buildman, since it does not use |
170 | # them. For example, Series-version is not useful for buildman. On the | |
171 | # other hand conflicting tags will cause an error. So allow later tags | |
172 | # to overwrite earlier ones by setting allow_overwrite=True | |
fea5858e | 173 | if options.branch: |
3b74ba5f SG |
174 | if count == -1: |
175 | range_expr = gitutil.GetRangeInBranch(options.git_dir, | |
176 | options.branch) | |
177 | upstream_commit = gitutil.GetUpstream(options.git_dir, | |
178 | options.branch) | |
179 | series = patchstream.GetMetaDataForList(upstream_commit, | |
950a2313 | 180 | options.git_dir, 1, series=None, allow_overwrite=True) |
3b74ba5f | 181 | |
3b74ba5f | 182 | series = patchstream.GetMetaDataForList(range_expr, |
950a2313 | 183 | options.git_dir, None, series, allow_overwrite=True) |
3b74ba5f SG |
184 | else: |
185 | # Honour the count | |
186 | series = patchstream.GetMetaDataForList(options.branch, | |
950a2313 | 187 | options.git_dir, count, series=None, allow_overwrite=True) |
fea5858e SG |
188 | else: |
189 | series = None | |
e5a0e5d8 | 190 | options.verbose = True |
58d818f1 SG |
191 | if not options.summary: |
192 | options.show_errors = True | |
fc3fe1c2 SG |
193 | |
194 | # By default we have one thread per CPU. But if there are not enough jobs | |
195 | # we can have fewer threads and use a high '-j' value for make. | |
196 | if not options.threads: | |
197 | options.threads = min(multiprocessing.cpu_count(), len(selected)) | |
198 | if not options.jobs: | |
199 | options.jobs = max(1, (multiprocessing.cpu_count() + | |
200 | len(selected) - 1) / len(selected)) | |
201 | ||
202 | if not options.step: | |
203 | options.step = len(series.commits) - 1 | |
204 | ||
99796923 MY |
205 | gnu_make = command.Output(os.path.join(options.git, |
206 | 'scripts/show-gnu-make')).rstrip() | |
207 | if not gnu_make: | |
31e2141d | 208 | sys.exit('GNU Make not found') |
99796923 | 209 | |
fc3fe1c2 | 210 | # Create a new builder with the selected options |
fea5858e | 211 | if options.branch: |
f7582ce8 | 212 | dirname = options.branch.replace('/', '_') |
fea5858e SG |
213 | else: |
214 | dirname = 'current' | |
215 | output_dir = os.path.join(options.output_dir, dirname) | |
883a321a SG |
216 | if clean_dir and os.path.exists(output_dir): |
217 | shutil.rmtree(output_dir) | |
fc3fe1c2 | 218 | builder = Builder(toolchains, output_dir, options.git_dir, |
99796923 | 219 | options.threads, options.jobs, gnu_make=gnu_make, checkout=True, |
fc3fe1c2 SG |
220 | show_unknown=options.show_unknown, step=options.step) |
221 | builder.force_config_on_failure = not options.quick | |
d4144e45 SG |
222 | if make_func: |
223 | builder.do_make = make_func | |
fc3fe1c2 SG |
224 | |
225 | # For a dry run, just show our actions as a sanity check | |
226 | if options.dry_run: | |
227 | ShowActions(series, why_selected, selected, builder, options) | |
228 | else: | |
229 | builder.force_build = options.force_build | |
4266dc28 | 230 | builder.force_build_failures = options.force_build_failures |
97e91526 | 231 | builder.force_reconfig = options.force_reconfig |
189a4968 | 232 | builder.in_tree = options.in_tree |
fc3fe1c2 SG |
233 | |
234 | # Work out which boards to build | |
235 | board_selected = boards.GetSelectedDict() | |
236 | ||
fea5858e SG |
237 | if series: |
238 | commits = series.commits | |
883a321a SG |
239 | # Number the commits for test purposes |
240 | for commit in range(len(commits)): | |
241 | commits[commit].sequence = commit | |
fea5858e SG |
242 | else: |
243 | commits = None | |
244 | ||
d4144e45 SG |
245 | Print(GetActionSummary(options.summary, commits, board_selected, |
246 | options)) | |
fc3fe1c2 | 247 | |
7798e228 SG |
248 | # We can't show function sizes without board details at present |
249 | if options.show_bloat: | |
250 | options.show_detail = True | |
b2ea7ab2 | 251 | builder.SetDisplayOptions(options.show_errors, options.show_sizes, |
ed966657 SG |
252 | options.show_detail, options.show_bloat, |
253 | options.list_error_boards) | |
fc3fe1c2 | 254 | if options.summary: |
b2ea7ab2 | 255 | builder.ShowSummary(commits, board_selected) |
fc3fe1c2 | 256 | else: |
2c3deb97 | 257 | fail, warned = builder.BuildBoards(commits, board_selected, |
e5a0e5d8 | 258 | options.keep_outputs, options.verbose) |
2c3deb97 SG |
259 | if fail: |
260 | return 128 | |
261 | elif warned: | |
262 | return 129 | |
263 | return 0 |