]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
3 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> | |
4 | # | |
1a459660 | 5 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
6 | # |
7 | ||
8 | import collections | |
fc3fe1c2 SG |
9 | from datetime import datetime, timedelta |
10 | import glob | |
11 | import os | |
12 | import re | |
13 | import Queue | |
14 | import shutil | |
15 | import string | |
16 | import sys | |
fc3fe1c2 SG |
17 | import time |
18 | ||
190064b4 | 19 | import builderthread |
fc3fe1c2 SG |
20 | import command |
21 | import gitutil | |
22 | import terminal | |
4653a882 | 23 | from terminal import Print |
fc3fe1c2 SG |
24 | import toolchain |
25 | ||
26 | ||
27 | """ | |
28 | Theory of Operation | |
29 | ||
30 | Please see README for user documentation, and you should be familiar with | |
31 | that before trying to make sense of this. | |
32 | ||
33 | Buildman works by keeping the machine as busy as possible, building different | |
34 | commits for different boards on multiple CPUs at once. | |
35 | ||
36 | The source repo (self.git_dir) contains all the commits to be built. Each | |
37 | thread works on a single board at a time. It checks out the first commit, | |
38 | configures it for that board, then builds it. Then it checks out the next | |
39 | commit and builds it (typically without re-configuring). When it runs out | |
40 | of commits, it gets another job from the builder and starts again with that | |
41 | board. | |
42 | ||
43 | Clearly the builder threads could work either way - they could check out a | |
44 | commit and then built it for all boards. Using separate directories for each | |
45 | commit/board pair they could leave their build product around afterwards | |
46 | also. | |
47 | ||
48 | The intent behind building a single board for multiple commits, is to make | |
49 | use of incremental builds. Since each commit is built incrementally from | |
50 | the previous one, builds are faster. Reconfiguring for a different board | |
51 | removes all intermediate object files. | |
52 | ||
53 | Many threads can be working at once, but each has its own working directory. | |
54 | When a thread finishes a build, it puts the output files into a result | |
55 | directory. | |
56 | ||
57 | The base directory used by buildman is normally '../<branch>', i.e. | |
58 | a directory higher than the source repository and named after the branch | |
59 | being built. | |
60 | ||
61 | Within the base directory, we have one subdirectory for each commit. Within | |
62 | that is one subdirectory for each board. Within that is the build output for | |
63 | that commit/board combination. | |
64 | ||
65 | Buildman also create working directories for each thread, in a .bm-work/ | |
66 | subdirectory in the base dir. | |
67 | ||
68 | As an example, say we are building branch 'us-net' for boards 'sandbox' and | |
69 | 'seaboard', and say that us-net has two commits. We will have directories | |
70 | like this: | |
71 | ||
72 | us-net/ base directory | |
73 | 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ | |
74 | sandbox/ | |
75 | u-boot.bin | |
76 | seaboard/ | |
77 | u-boot.bin | |
78 | 02_of_02_g4ed4ebc_net--Check-tftp-comp/ | |
79 | sandbox/ | |
80 | u-boot.bin | |
81 | seaboard/ | |
82 | u-boot.bin | |
83 | .bm-work/ | |
84 | 00/ working directory for thread 0 (contains source checkout) | |
85 | build/ build output | |
86 | 01/ working directory for thread 1 | |
87 | build/ build output | |
88 | ... | |
89 | u-boot/ source directory | |
90 | .git/ repository | |
91 | """ | |
92 | ||
93 | # Possible build outcomes | |
94 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) | |
95 | ||
96 | # Translate a commit subject into a valid filename | |
97 | trans_valid_chars = string.maketrans("/: ", "---") | |
98 | ||
99 | ||
fc3fe1c2 SG |
100 | class Builder: |
101 | """Class for building U-Boot for a particular commit. | |
102 | ||
103 | Public members: (many should ->private) | |
104 | active: True if the builder is active and has not been stopped | |
105 | already_done: Number of builds already completed | |
106 | base_dir: Base directory to use for builder | |
107 | checkout: True to check out source, False to skip that step. | |
108 | This is used for testing. | |
109 | col: terminal.Color() object | |
110 | count: Number of commits to build | |
111 | do_make: Method to call to invoke Make | |
112 | fail: Number of builds that failed due to error | |
113 | force_build: Force building even if a build already exists | |
114 | force_config_on_failure: If a commit fails for a board, disable | |
115 | incremental building for the next commit we build for that | |
116 | board, so that we will see all warnings/errors again. | |
4266dc28 SG |
117 | force_build_failures: If a previously-built build (i.e. built on |
118 | a previous run of buildman) is marked as failed, rebuild it. | |
fc3fe1c2 SG |
119 | git_dir: Git directory containing source repository |
120 | last_line_len: Length of the last line we printed (used for erasing | |
121 | it with new progress information) | |
122 | num_jobs: Number of jobs to run at once (passed to make as -j) | |
123 | num_threads: Number of builder threads to run | |
124 | out_queue: Queue of results to process | |
125 | re_make_err: Compiled regular expression for ignore_lines | |
126 | queue: Queue of jobs to run | |
127 | threads: List of active threads | |
128 | toolchains: Toolchains object to use for building | |
129 | upto: Current commit number we are building (0.count-1) | |
130 | warned: Number of builds that produced at least one warning | |
97e91526 SG |
131 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
132 | incremental building, where buildman reconfigures on the first | |
133 | commit for a baord, and then just does an incremental build for | |
134 | the following commits. In fact buildman will reconfigure and | |
135 | retry for any failing commits, so generally the only effect of | |
136 | this option is to slow things down. | |
189a4968 SG |
137 | in_tree: Build U-Boot in-tree instead of specifying an output |
138 | directory separate from the source code. This option is really | |
139 | only useful for testing in-tree builds. | |
fc3fe1c2 SG |
140 | |
141 | Private members: | |
142 | _base_board_dict: Last-summarised Dict of boards | |
143 | _base_err_lines: Last-summarised list of errors | |
e30965db | 144 | _base_warn_lines: Last-summarised list of warnings |
fc3fe1c2 SG |
145 | _build_period_us: Time taken for a single build (float object). |
146 | _complete_delay: Expected delay until completion (timedelta) | |
147 | _next_delay_update: Next time we plan to display a progress update | |
148 | (datatime) | |
149 | _show_unknown: Show unknown boards (those not built) in summary | |
150 | _timestamps: List of timestamps for the completion of the last | |
151 | last _timestamp_count builds. Each is a datetime object. | |
152 | _timestamp_count: Number of timestamps to keep in our list. | |
153 | _working_dir: Base working directory containing all threads | |
154 | """ | |
155 | class Outcome: | |
156 | """Records a build outcome for a single make invocation | |
157 | ||
158 | Public Members: | |
159 | rc: Outcome value (OUTCOME_...) | |
160 | err_lines: List of error lines or [] if none | |
161 | sizes: Dictionary of image size information, keyed by filename | |
162 | - Each value is itself a dictionary containing | |
163 | values for 'text', 'data' and 'bss', being the integer | |
164 | size in bytes of each section. | |
165 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each | |
166 | value is itself a dictionary: | |
167 | key: function name | |
168 | value: Size of function in bytes | |
169 | """ | |
170 | def __init__(self, rc, err_lines, sizes, func_sizes): | |
171 | self.rc = rc | |
172 | self.err_lines = err_lines | |
173 | self.sizes = sizes | |
174 | self.func_sizes = func_sizes | |
175 | ||
176 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, | |
99796923 | 177 | gnu_make='make', checkout=True, show_unknown=True, step=1): |
fc3fe1c2 SG |
178 | """Create a new Builder object |
179 | ||
180 | Args: | |
181 | toolchains: Toolchains object to use for building | |
182 | base_dir: Base directory to use for builder | |
183 | git_dir: Git directory containing source repository | |
184 | num_threads: Number of builder threads to run | |
185 | num_jobs: Number of jobs to run at once (passed to make as -j) | |
99796923 | 186 | gnu_make: the command name of GNU Make. |
fc3fe1c2 SG |
187 | checkout: True to check out source, False to skip that step. |
188 | This is used for testing. | |
189 | show_unknown: Show unknown boards (those not built) in summary | |
190 | step: 1 to process every commit, n to process every nth commit | |
191 | """ | |
192 | self.toolchains = toolchains | |
193 | self.base_dir = base_dir | |
194 | self._working_dir = os.path.join(base_dir, '.bm-work') | |
195 | self.threads = [] | |
196 | self.active = True | |
197 | self.do_make = self.Make | |
99796923 | 198 | self.gnu_make = gnu_make |
fc3fe1c2 SG |
199 | self.checkout = checkout |
200 | self.num_threads = num_threads | |
201 | self.num_jobs = num_jobs | |
202 | self.already_done = 0 | |
203 | self.force_build = False | |
204 | self.git_dir = git_dir | |
205 | self._show_unknown = show_unknown | |
206 | self._timestamp_count = 10 | |
207 | self._build_period_us = None | |
208 | self._complete_delay = None | |
209 | self._next_delay_update = datetime.now() | |
210 | self.force_config_on_failure = True | |
4266dc28 | 211 | self.force_build_failures = False |
97e91526 | 212 | self.force_reconfig = False |
fc3fe1c2 | 213 | self._step = step |
189a4968 | 214 | self.in_tree = False |
28370c1b | 215 | self._error_lines = 0 |
fc3fe1c2 SG |
216 | |
217 | self.col = terminal.Color() | |
218 | ||
e30965db SG |
219 | self._re_function = re.compile('(.*): In function.*') |
220 | self._re_files = re.compile('In file included from.*') | |
221 | self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') | |
222 | self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') | |
223 | ||
fc3fe1c2 SG |
224 | self.queue = Queue.Queue() |
225 | self.out_queue = Queue.Queue() | |
226 | for i in range(self.num_threads): | |
190064b4 | 227 | t = builderthread.BuilderThread(self, i) |
fc3fe1c2 SG |
228 | t.setDaemon(True) |
229 | t.start() | |
230 | self.threads.append(t) | |
231 | ||
232 | self.last_line_len = 0 | |
190064b4 | 233 | t = builderthread.ResultThread(self) |
fc3fe1c2 SG |
234 | t.setDaemon(True) |
235 | t.start() | |
236 | self.threads.append(t) | |
237 | ||
238 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] | |
239 | self.re_make_err = re.compile('|'.join(ignore_lines)) | |
240 | ||
241 | def __del__(self): | |
242 | """Get rid of all threads created by the builder""" | |
243 | for t in self.threads: | |
244 | del t | |
245 | ||
b2ea7ab2 | 246 | def SetDisplayOptions(self, show_errors=False, show_sizes=False, |
ed966657 SG |
247 | show_detail=False, show_bloat=False, |
248 | list_error_boards=False): | |
b2ea7ab2 SG |
249 | """Setup display options for the builder. |
250 | ||
251 | show_errors: True to show summarised error/warning info | |
252 | show_sizes: Show size deltas | |
253 | show_detail: Show detail for each board | |
254 | show_bloat: Show detail for each function | |
ed966657 | 255 | list_error_boards: Show the boards which caused each error/warning |
b2ea7ab2 SG |
256 | """ |
257 | self._show_errors = show_errors | |
258 | self._show_sizes = show_sizes | |
259 | self._show_detail = show_detail | |
260 | self._show_bloat = show_bloat | |
ed966657 | 261 | self._list_error_boards = list_error_boards |
b2ea7ab2 | 262 | |
fc3fe1c2 SG |
263 | def _AddTimestamp(self): |
264 | """Add a new timestamp to the list and record the build period. | |
265 | ||
266 | The build period is the length of time taken to perform a single | |
267 | build (one board, one commit). | |
268 | """ | |
269 | now = datetime.now() | |
270 | self._timestamps.append(now) | |
271 | count = len(self._timestamps) | |
272 | delta = self._timestamps[-1] - self._timestamps[0] | |
273 | seconds = delta.total_seconds() | |
274 | ||
275 | # If we have enough data, estimate build period (time taken for a | |
276 | # single build) and therefore completion time. | |
277 | if count > 1 and self._next_delay_update < now: | |
278 | self._next_delay_update = now + timedelta(seconds=2) | |
279 | if seconds > 0: | |
280 | self._build_period = float(seconds) / count | |
281 | todo = self.count - self.upto | |
282 | self._complete_delay = timedelta(microseconds= | |
283 | self._build_period * todo * 1000000) | |
284 | # Round it | |
285 | self._complete_delay -= timedelta( | |
286 | microseconds=self._complete_delay.microseconds) | |
287 | ||
288 | if seconds > 60: | |
289 | self._timestamps.popleft() | |
290 | count -= 1 | |
291 | ||
292 | def ClearLine(self, length): | |
293 | """Clear any characters on the current line | |
294 | ||
295 | Make way for a new line of length 'length', by outputting enough | |
296 | spaces to clear out the old line. Then remember the new length for | |
297 | next time. | |
298 | ||
299 | Args: | |
300 | length: Length of new line, in characters | |
301 | """ | |
302 | if length < self.last_line_len: | |
4653a882 SG |
303 | Print(' ' * (self.last_line_len - length), newline=False) |
304 | Print('\r', newline=False) | |
fc3fe1c2 SG |
305 | self.last_line_len = length |
306 | sys.stdout.flush() | |
307 | ||
308 | def SelectCommit(self, commit, checkout=True): | |
309 | """Checkout the selected commit for this build | |
310 | """ | |
311 | self.commit = commit | |
312 | if checkout and self.checkout: | |
313 | gitutil.Checkout(commit.hash) | |
314 | ||
315 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): | |
316 | """Run make | |
317 | ||
318 | Args: | |
319 | commit: Commit object that is being built | |
320 | brd: Board object that is being built | |
fd18a89e | 321 | stage: Stage that we are at (mrproper, config, build) |
fc3fe1c2 SG |
322 | cwd: Directory where make should be run |
323 | args: Arguments to pass to make | |
324 | kwargs: Arguments to pass to command.RunPipe() | |
325 | """ | |
99796923 | 326 | cmd = [self.gnu_make] + list(args) |
fc3fe1c2 SG |
327 | result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
328 | cwd=cwd, raise_on_error=False, **kwargs) | |
329 | return result | |
330 | ||
331 | def ProcessResult(self, result): | |
332 | """Process the result of a build, showing progress information | |
333 | ||
334 | Args: | |
e5a0e5d8 SG |
335 | result: A CommandResult object, which indicates the result for |
336 | a single build | |
fc3fe1c2 SG |
337 | """ |
338 | col = terminal.Color() | |
339 | if result: | |
340 | target = result.brd.target | |
341 | ||
342 | if result.return_code < 0: | |
343 | self.active = False | |
344 | command.StopAll() | |
345 | return | |
346 | ||
347 | self.upto += 1 | |
348 | if result.return_code != 0: | |
349 | self.fail += 1 | |
350 | elif result.stderr: | |
351 | self.warned += 1 | |
352 | if result.already_done: | |
353 | self.already_done += 1 | |
e5a0e5d8 | 354 | if self._verbose: |
4653a882 | 355 | Print('\r', newline=False) |
e5a0e5d8 SG |
356 | self.ClearLine(0) |
357 | boards_selected = {target : result.brd} | |
358 | self.ResetResultSummary(boards_selected) | |
359 | self.ProduceResultSummary(result.commit_upto, self.commits, | |
360 | boards_selected) | |
fc3fe1c2 SG |
361 | else: |
362 | target = '(starting)' | |
363 | ||
364 | # Display separate counts for ok, warned and fail | |
365 | ok = self.upto - self.warned - self.fail | |
366 | line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) | |
367 | line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) | |
368 | line += self.col.Color(self.col.RED, '%5d' % self.fail) | |
369 | ||
370 | name = ' /%-5d ' % self.count | |
371 | ||
372 | # Add our current completion time estimate | |
373 | self._AddTimestamp() | |
374 | if self._complete_delay: | |
375 | name += '%s : ' % self._complete_delay | |
376 | # When building all boards for a commit, we can print a commit | |
377 | # progress message. | |
378 | if result and result.commit_upto is None: | |
379 | name += 'commit %2d/%-3d' % (self.commit_upto + 1, | |
380 | self.commit_count) | |
381 | ||
382 | name += target | |
4653a882 | 383 | Print(line + name, newline=False) |
e5a0e5d8 | 384 | length = 14 + len(name) |
fc3fe1c2 SG |
385 | self.ClearLine(length) |
386 | ||
387 | def _GetOutputDir(self, commit_upto): | |
388 | """Get the name of the output directory for a commit number | |
389 | ||
390 | The output directory is typically .../<branch>/<commit>. | |
391 | ||
392 | Args: | |
393 | commit_upto: Commit number to use (0..self.count-1) | |
394 | """ | |
fea5858e SG |
395 | if self.commits: |
396 | commit = self.commits[commit_upto] | |
397 | subject = commit.subject.translate(trans_valid_chars) | |
398 | commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, | |
399 | self.commit_count, commit.hash, subject[:20])) | |
400 | else: | |
401 | commit_dir = 'current' | |
fc3fe1c2 SG |
402 | output_dir = os.path.join(self.base_dir, commit_dir) |
403 | return output_dir | |
404 | ||
405 | def GetBuildDir(self, commit_upto, target): | |
406 | """Get the name of the build directory for a commit number | |
407 | ||
408 | The build directory is typically .../<branch>/<commit>/<target>. | |
409 | ||
410 | Args: | |
411 | commit_upto: Commit number to use (0..self.count-1) | |
412 | target: Target name | |
413 | """ | |
414 | output_dir = self._GetOutputDir(commit_upto) | |
415 | return os.path.join(output_dir, target) | |
416 | ||
417 | def GetDoneFile(self, commit_upto, target): | |
418 | """Get the name of the done file for a commit number | |
419 | ||
420 | Args: | |
421 | commit_upto: Commit number to use (0..self.count-1) | |
422 | target: Target name | |
423 | """ | |
424 | return os.path.join(self.GetBuildDir(commit_upto, target), 'done') | |
425 | ||
426 | def GetSizesFile(self, commit_upto, target): | |
427 | """Get the name of the sizes file for a commit number | |
428 | ||
429 | Args: | |
430 | commit_upto: Commit number to use (0..self.count-1) | |
431 | target: Target name | |
432 | """ | |
433 | return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') | |
434 | ||
435 | def GetFuncSizesFile(self, commit_upto, target, elf_fname): | |
436 | """Get the name of the funcsizes file for a commit number and ELF file | |
437 | ||
438 | Args: | |
439 | commit_upto: Commit number to use (0..self.count-1) | |
440 | target: Target name | |
441 | elf_fname: Filename of elf image | |
442 | """ | |
443 | return os.path.join(self.GetBuildDir(commit_upto, target), | |
444 | '%s.sizes' % elf_fname.replace('/', '-')) | |
445 | ||
446 | def GetObjdumpFile(self, commit_upto, target, elf_fname): | |
447 | """Get the name of the objdump file for a commit number and ELF file | |
448 | ||
449 | Args: | |
450 | commit_upto: Commit number to use (0..self.count-1) | |
451 | target: Target name | |
452 | elf_fname: Filename of elf image | |
453 | """ | |
454 | return os.path.join(self.GetBuildDir(commit_upto, target), | |
455 | '%s.objdump' % elf_fname.replace('/', '-')) | |
456 | ||
457 | def GetErrFile(self, commit_upto, target): | |
458 | """Get the name of the err file for a commit number | |
459 | ||
460 | Args: | |
461 | commit_upto: Commit number to use (0..self.count-1) | |
462 | target: Target name | |
463 | """ | |
464 | output_dir = self.GetBuildDir(commit_upto, target) | |
465 | return os.path.join(output_dir, 'err') | |
466 | ||
467 | def FilterErrors(self, lines): | |
468 | """Filter out errors in which we have no interest | |
469 | ||
470 | We should probably use map(). | |
471 | ||
472 | Args: | |
473 | lines: List of error lines, each a string | |
474 | Returns: | |
475 | New list with only interesting lines included | |
476 | """ | |
477 | out_lines = [] | |
478 | for line in lines: | |
479 | if not self.re_make_err.search(line): | |
480 | out_lines.append(line) | |
481 | return out_lines | |
482 | ||
483 | def ReadFuncSizes(self, fname, fd): | |
484 | """Read function sizes from the output of 'nm' | |
485 | ||
486 | Args: | |
487 | fd: File containing data to read | |
488 | fname: Filename we are reading from (just for errors) | |
489 | ||
490 | Returns: | |
491 | Dictionary containing size of each function in bytes, indexed by | |
492 | function name. | |
493 | """ | |
494 | sym = {} | |
495 | for line in fd.readlines(): | |
496 | try: | |
497 | size, type, name = line[:-1].split() | |
498 | except: | |
4653a882 | 499 | Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) |
fc3fe1c2 SG |
500 | continue |
501 | if type in 'tTdDbB': | |
502 | # function names begin with '.' on 64-bit powerpc | |
503 | if '.' in name[1:]: | |
504 | name = 'static.' + name.split('.')[0] | |
505 | sym[name] = sym.get(name, 0) + int(size, 16) | |
506 | return sym | |
507 | ||
508 | def GetBuildOutcome(self, commit_upto, target, read_func_sizes): | |
509 | """Work out the outcome of a build. | |
510 | ||
511 | Args: | |
512 | commit_upto: Commit number to check (0..n-1) | |
513 | target: Target board to check | |
514 | read_func_sizes: True to read function size information | |
515 | ||
516 | Returns: | |
517 | Outcome object | |
518 | """ | |
519 | done_file = self.GetDoneFile(commit_upto, target) | |
520 | sizes_file = self.GetSizesFile(commit_upto, target) | |
521 | sizes = {} | |
522 | func_sizes = {} | |
523 | if os.path.exists(done_file): | |
524 | with open(done_file, 'r') as fd: | |
525 | return_code = int(fd.readline()) | |
526 | err_lines = [] | |
527 | err_file = self.GetErrFile(commit_upto, target) | |
528 | if os.path.exists(err_file): | |
529 | with open(err_file, 'r') as fd: | |
530 | err_lines = self.FilterErrors(fd.readlines()) | |
531 | ||
532 | # Decide whether the build was ok, failed or created warnings | |
533 | if return_code: | |
534 | rc = OUTCOME_ERROR | |
535 | elif len(err_lines): | |
536 | rc = OUTCOME_WARNING | |
537 | else: | |
538 | rc = OUTCOME_OK | |
539 | ||
540 | # Convert size information to our simple format | |
541 | if os.path.exists(sizes_file): | |
542 | with open(sizes_file, 'r') as fd: | |
543 | for line in fd.readlines(): | |
544 | values = line.split() | |
545 | rodata = 0 | |
546 | if len(values) > 6: | |
547 | rodata = int(values[6], 16) | |
548 | size_dict = { | |
549 | 'all' : int(values[0]) + int(values[1]) + | |
550 | int(values[2]), | |
551 | 'text' : int(values[0]) - rodata, | |
552 | 'data' : int(values[1]), | |
553 | 'bss' : int(values[2]), | |
554 | 'rodata' : rodata, | |
555 | } | |
556 | sizes[values[5]] = size_dict | |
557 | ||
558 | if read_func_sizes: | |
559 | pattern = self.GetFuncSizesFile(commit_upto, target, '*') | |
560 | for fname in glob.glob(pattern): | |
561 | with open(fname, 'r') as fd: | |
562 | dict_name = os.path.basename(fname).replace('.sizes', | |
563 | '') | |
564 | func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) | |
565 | ||
566 | return Builder.Outcome(rc, err_lines, sizes, func_sizes) | |
567 | ||
568 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) | |
569 | ||
570 | def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): | |
571 | """Calculate a summary of the results of building a commit. | |
572 | ||
573 | Args: | |
574 | board_selected: Dict containing boards to summarise | |
575 | commit_upto: Commit number to summarize (0..self.count-1) | |
576 | read_func_sizes: True to read function size information | |
577 | ||
578 | Returns: | |
579 | Tuple: | |
580 | Dict containing boards which passed building this commit. | |
581 | keyed by board.target | |
e30965db | 582 | List containing a summary of error lines |
ed966657 SG |
583 | Dict keyed by error line, containing a list of the Board |
584 | objects with that error | |
e30965db SG |
585 | List containing a summary of warning lines |
586 | Dict keyed by error line, containing a list of the Board | |
587 | objects with that warning | |
fc3fe1c2 | 588 | """ |
e30965db SG |
589 | def AddLine(lines_summary, lines_boards, line, board): |
590 | line = line.rstrip() | |
591 | if line in lines_boards: | |
592 | lines_boards[line].append(board) | |
593 | else: | |
594 | lines_boards[line] = [board] | |
595 | lines_summary.append(line) | |
596 | ||
fc3fe1c2 SG |
597 | board_dict = {} |
598 | err_lines_summary = [] | |
ed966657 | 599 | err_lines_boards = {} |
e30965db SG |
600 | warn_lines_summary = [] |
601 | warn_lines_boards = {} | |
fc3fe1c2 SG |
602 | |
603 | for board in boards_selected.itervalues(): | |
604 | outcome = self.GetBuildOutcome(commit_upto, board.target, | |
605 | read_func_sizes) | |
606 | board_dict[board.target] = outcome | |
e30965db SG |
607 | last_func = None |
608 | last_was_warning = False | |
609 | for line in outcome.err_lines: | |
610 | if line: | |
611 | if (self._re_function.match(line) or | |
612 | self._re_files.match(line)): | |
613 | last_func = line | |
ed966657 | 614 | else: |
e30965db SG |
615 | is_warning = self._re_warning.match(line) |
616 | is_note = self._re_note.match(line) | |
617 | if is_warning or (last_was_warning and is_note): | |
618 | if last_func: | |
619 | AddLine(warn_lines_summary, warn_lines_boards, | |
620 | last_func, board) | |
621 | AddLine(warn_lines_summary, warn_lines_boards, | |
622 | line, board) | |
623 | else: | |
624 | if last_func: | |
625 | AddLine(err_lines_summary, err_lines_boards, | |
626 | last_func, board) | |
627 | AddLine(err_lines_summary, err_lines_boards, | |
628 | line, board) | |
629 | last_was_warning = is_warning | |
630 | last_func = None | |
631 | return (board_dict, err_lines_summary, err_lines_boards, | |
632 | warn_lines_summary, warn_lines_boards) | |
fc3fe1c2 SG |
633 | |
634 | def AddOutcome(self, board_dict, arch_list, changes, char, color): | |
635 | """Add an output to our list of outcomes for each architecture | |
636 | ||
637 | This simple function adds failing boards (changes) to the | |
638 | relevant architecture string, so we can print the results out | |
639 | sorted by architecture. | |
640 | ||
641 | Args: | |
642 | board_dict: Dict containing all boards | |
643 | arch_list: Dict keyed by arch name. Value is a string containing | |
644 | a list of board names which failed for that arch. | |
645 | changes: List of boards to add to arch_list | |
646 | color: terminal.Colour object | |
647 | """ | |
648 | done_arch = {} | |
649 | for target in changes: | |
650 | if target in board_dict: | |
651 | arch = board_dict[target].arch | |
652 | else: | |
653 | arch = 'unknown' | |
654 | str = self.col.Color(color, ' ' + target) | |
655 | if not arch in done_arch: | |
656 | str = self.col.Color(color, char) + ' ' + str | |
657 | done_arch[arch] = True | |
658 | if not arch in arch_list: | |
659 | arch_list[arch] = str | |
660 | else: | |
661 | arch_list[arch] += str | |
662 | ||
663 | ||
664 | def ColourNum(self, num): | |
665 | color = self.col.RED if num > 0 else self.col.GREEN | |
666 | if num == 0: | |
667 | return '0' | |
668 | return self.col.Color(color, str(num)) | |
669 | ||
670 | def ResetResultSummary(self, board_selected): | |
671 | """Reset the results summary ready for use. | |
672 | ||
673 | Set up the base board list to be all those selected, and set the | |
674 | error lines to empty. | |
675 | ||
676 | Following this, calls to PrintResultSummary() will use this | |
677 | information to work out what has changed. | |
678 | ||
679 | Args: | |
680 | board_selected: Dict containing boards to summarise, keyed by | |
681 | board.target | |
682 | """ | |
683 | self._base_board_dict = {} | |
684 | for board in board_selected: | |
685 | self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) | |
686 | self._base_err_lines = [] | |
e30965db SG |
687 | self._base_warn_lines = [] |
688 | self._base_err_line_boards = {} | |
689 | self._base_warn_line_boards = {} | |
fc3fe1c2 SG |
690 | |
691 | def PrintFuncSizeDetail(self, fname, old, new): | |
692 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 | |
693 | delta, common = [], {} | |
694 | ||
695 | for a in old: | |
696 | if a in new: | |
697 | common[a] = 1 | |
698 | ||
699 | for name in old: | |
700 | if name not in common: | |
701 | remove += 1 | |
702 | down += old[name] | |
703 | delta.append([-old[name], name]) | |
704 | ||
705 | for name in new: | |
706 | if name not in common: | |
707 | add += 1 | |
708 | up += new[name] | |
709 | delta.append([new[name], name]) | |
710 | ||
711 | for name in common: | |
712 | diff = new.get(name, 0) - old.get(name, 0) | |
713 | if diff > 0: | |
714 | grow, up = grow + 1, up + diff | |
715 | elif diff < 0: | |
716 | shrink, down = shrink + 1, down - diff | |
717 | delta.append([diff, name]) | |
718 | ||
719 | delta.sort() | |
720 | delta.reverse() | |
721 | ||
722 | args = [add, -remove, grow, -shrink, up, -down, up - down] | |
723 | if max(args) == 0: | |
724 | return | |
725 | args = [self.ColourNum(x) for x in args] | |
726 | indent = ' ' * 15 | |
4653a882 SG |
727 | Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
728 | tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) | |
729 | Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', | |
730 | 'delta')) | |
fc3fe1c2 SG |
731 | for diff, name in delta: |
732 | if diff: | |
733 | color = self.col.RED if diff > 0 else self.col.GREEN | |
734 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, | |
735 | old.get(name, '-'), new.get(name,'-'), diff) | |
4653a882 | 736 | Print(msg, colour=color) |
fc3fe1c2 SG |
737 | |
738 | ||
739 | def PrintSizeDetail(self, target_list, show_bloat): | |
740 | """Show details size information for each board | |
741 | ||
742 | Args: | |
743 | target_list: List of targets, each a dict containing: | |
744 | 'target': Target name | |
745 | 'total_diff': Total difference in bytes across all areas | |
746 | <part_name>: Difference for that part | |
747 | show_bloat: Show detail for each function | |
748 | """ | |
749 | targets_by_diff = sorted(target_list, reverse=True, | |
750 | key=lambda x: x['_total_diff']) | |
751 | for result in targets_by_diff: | |
752 | printed_target = False | |
753 | for name in sorted(result): | |
754 | diff = result[name] | |
755 | if name.startswith('_'): | |
756 | continue | |
757 | if diff != 0: | |
758 | color = self.col.RED if diff > 0 else self.col.GREEN | |
759 | msg = ' %s %+d' % (name, diff) | |
760 | if not printed_target: | |
4653a882 SG |
761 | Print('%10s %-15s:' % ('', result['_target']), |
762 | newline=False) | |
fc3fe1c2 | 763 | printed_target = True |
4653a882 | 764 | Print(msg, colour=color, newline=False) |
fc3fe1c2 | 765 | if printed_target: |
4653a882 | 766 | Print() |
fc3fe1c2 SG |
767 | if show_bloat: |
768 | target = result['_target'] | |
769 | outcome = result['_outcome'] | |
770 | base_outcome = self._base_board_dict[target] | |
771 | for fname in outcome.func_sizes: | |
772 | self.PrintFuncSizeDetail(fname, | |
773 | base_outcome.func_sizes[fname], | |
774 | outcome.func_sizes[fname]) | |
775 | ||
776 | ||
777 | def PrintSizeSummary(self, board_selected, board_dict, show_detail, | |
778 | show_bloat): | |
779 | """Print a summary of image sizes broken down by section. | |
780 | ||
781 | The summary takes the form of one line per architecture. The | |
782 | line contains deltas for each of the sections (+ means the section | |
783 | got bigger, - means smaller). The nunmbers are the average number | |
784 | of bytes that a board in this section increased by. | |
785 | ||
786 | For example: | |
787 | powerpc: (622 boards) text -0.0 | |
788 | arm: (285 boards) text -0.0 | |
789 | nds32: (3 boards) text -8.0 | |
790 | ||
791 | Args: | |
792 | board_selected: Dict containing boards to summarise, keyed by | |
793 | board.target | |
794 | board_dict: Dict containing boards for which we built this | |
795 | commit, keyed by board.target. The value is an Outcome object. | |
796 | show_detail: Show detail for each board | |
797 | show_bloat: Show detail for each function | |
798 | """ | |
799 | arch_list = {} | |
800 | arch_count = {} | |
801 | ||
802 | # Calculate changes in size for different image parts | |
803 | # The previous sizes are in Board.sizes, for each board | |
804 | for target in board_dict: | |
805 | if target not in board_selected: | |
806 | continue | |
807 | base_sizes = self._base_board_dict[target].sizes | |
808 | outcome = board_dict[target] | |
809 | sizes = outcome.sizes | |
810 | ||
811 | # Loop through the list of images, creating a dict of size | |
812 | # changes for each image/part. We end up with something like | |
813 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} | |
814 | # which means that U-Boot data increased by 5 bytes and SPL | |
815 | # text decreased by 4. | |
816 | err = {'_target' : target} | |
817 | for image in sizes: | |
818 | if image in base_sizes: | |
819 | base_image = base_sizes[image] | |
820 | # Loop through the text, data, bss parts | |
821 | for part in sorted(sizes[image]): | |
822 | diff = sizes[image][part] - base_image[part] | |
823 | col = None | |
824 | if diff: | |
825 | if image == 'u-boot': | |
826 | name = part | |
827 | else: | |
828 | name = image + ':' + part | |
829 | err[name] = diff | |
830 | arch = board_selected[target].arch | |
831 | if not arch in arch_count: | |
832 | arch_count[arch] = 1 | |
833 | else: | |
834 | arch_count[arch] += 1 | |
835 | if not sizes: | |
836 | pass # Only add to our list when we have some stats | |
837 | elif not arch in arch_list: | |
838 | arch_list[arch] = [err] | |
839 | else: | |
840 | arch_list[arch].append(err) | |
841 | ||
842 | # We now have a list of image size changes sorted by arch | |
843 | # Print out a summary of these | |
844 | for arch, target_list in arch_list.iteritems(): | |
845 | # Get total difference for each type | |
846 | totals = {} | |
847 | for result in target_list: | |
848 | total = 0 | |
849 | for name, diff in result.iteritems(): | |
850 | if name.startswith('_'): | |
851 | continue | |
852 | total += diff | |
853 | if name in totals: | |
854 | totals[name] += diff | |
855 | else: | |
856 | totals[name] = diff | |
857 | result['_total_diff'] = total | |
858 | result['_outcome'] = board_dict[result['_target']] | |
859 | ||
860 | count = len(target_list) | |
861 | printed_arch = False | |
862 | for name in sorted(totals): | |
863 | diff = totals[name] | |
864 | if diff: | |
865 | # Display the average difference in this name for this | |
866 | # architecture | |
867 | avg_diff = float(diff) / count | |
868 | color = self.col.RED if avg_diff > 0 else self.col.GREEN | |
869 | msg = ' %s %+1.1f' % (name, avg_diff) | |
870 | if not printed_arch: | |
4653a882 SG |
871 | Print('%10s: (for %d/%d boards)' % (arch, count, |
872 | arch_count[arch]), newline=False) | |
fc3fe1c2 | 873 | printed_arch = True |
4653a882 | 874 | Print(msg, colour=color, newline=False) |
fc3fe1c2 SG |
875 | |
876 | if printed_arch: | |
4653a882 | 877 | Print() |
fc3fe1c2 SG |
878 | if show_detail: |
879 | self.PrintSizeDetail(target_list, show_bloat) | |
880 | ||
881 | ||
882 | def PrintResultSummary(self, board_selected, board_dict, err_lines, | |
e30965db SG |
883 | err_line_boards, warn_lines, warn_line_boards, |
884 | show_sizes, show_detail, show_bloat): | |
fc3fe1c2 SG |
885 | """Compare results with the base results and display delta. |
886 | ||
887 | Only boards mentioned in board_selected will be considered. This | |
888 | function is intended to be called repeatedly with the results of | |
889 | each commit. It therefore shows a 'diff' between what it saw in | |
890 | the last call and what it sees now. | |
891 | ||
892 | Args: | |
893 | board_selected: Dict containing boards to summarise, keyed by | |
894 | board.target | |
895 | board_dict: Dict containing boards for which we built this | |
896 | commit, keyed by board.target. The value is an Outcome object. | |
897 | err_lines: A list of errors for this commit, or [] if there is | |
898 | none, or we don't want to print errors | |
ed966657 SG |
899 | err_line_boards: Dict keyed by error line, containing a list of |
900 | the Board objects with that error | |
e30965db SG |
901 | warn_lines: A list of warnings for this commit, or [] if there is |
902 | none, or we don't want to print errors | |
903 | warn_line_boards: Dict keyed by warning line, containing a list of | |
904 | the Board objects with that warning | |
fc3fe1c2 SG |
905 | show_sizes: Show image size deltas |
906 | show_detail: Show detail for each board | |
907 | show_bloat: Show detail for each function | |
908 | """ | |
e30965db | 909 | def _BoardList(line, line_boards): |
ed966657 SG |
910 | """Helper function to get a line of boards containing a line |
911 | ||
912 | Args: | |
913 | line: Error line to search for | |
914 | Return: | |
915 | String containing a list of boards with that error line, or | |
916 | '' if the user has not requested such a list | |
917 | """ | |
918 | if self._list_error_boards: | |
919 | names = [] | |
e30965db | 920 | for board in line_boards[line]: |
f66153be SG |
921 | if not board.target in names: |
922 | names.append(board.target) | |
ed966657 SG |
923 | names_str = '(%s) ' % ','.join(names) |
924 | else: | |
925 | names_str = '' | |
926 | return names_str | |
927 | ||
e30965db SG |
928 | def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, |
929 | char): | |
930 | better_lines = [] | |
931 | worse_lines = [] | |
932 | for line in lines: | |
933 | if line not in base_lines: | |
934 | worse_lines.append(char + '+' + | |
935 | _BoardList(line, line_boards) + line) | |
936 | for line in base_lines: | |
937 | if line not in lines: | |
938 | better_lines.append(char + '-' + | |
939 | _BoardList(line, base_line_boards) + line) | |
940 | return better_lines, worse_lines | |
941 | ||
fc3fe1c2 SG |
942 | better = [] # List of boards fixed since last commit |
943 | worse = [] # List of new broken boards since last commit | |
944 | new = [] # List of boards that didn't exist last time | |
945 | unknown = [] # List of boards that were not built | |
946 | ||
947 | for target in board_dict: | |
948 | if target not in board_selected: | |
949 | continue | |
950 | ||
951 | # If the board was built last time, add its outcome to a list | |
952 | if target in self._base_board_dict: | |
953 | base_outcome = self._base_board_dict[target].rc | |
954 | outcome = board_dict[target] | |
955 | if outcome.rc == OUTCOME_UNKNOWN: | |
956 | unknown.append(target) | |
957 | elif outcome.rc < base_outcome: | |
958 | better.append(target) | |
959 | elif outcome.rc > base_outcome: | |
960 | worse.append(target) | |
961 | else: | |
962 | new.append(target) | |
963 | ||
964 | # Get a list of errors that have appeared, and disappeared | |
e30965db SG |
965 | better_err, worse_err = _CalcErrorDelta(self._base_err_lines, |
966 | self._base_err_line_boards, err_lines, err_line_boards, '') | |
967 | better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, | |
968 | self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') | |
fc3fe1c2 SG |
969 | |
970 | # Display results by arch | |
e30965db SG |
971 | if (better or worse or unknown or new or worse_err or better_err |
972 | or worse_warn or better_warn): | |
fc3fe1c2 SG |
973 | arch_list = {} |
974 | self.AddOutcome(board_selected, arch_list, better, '', | |
975 | self.col.GREEN) | |
976 | self.AddOutcome(board_selected, arch_list, worse, '+', | |
977 | self.col.RED) | |
978 | self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) | |
979 | if self._show_unknown: | |
980 | self.AddOutcome(board_selected, arch_list, unknown, '?', | |
981 | self.col.MAGENTA) | |
982 | for arch, target_list in arch_list.iteritems(): | |
4653a882 | 983 | Print('%10s: %s' % (arch, target_list)) |
28370c1b | 984 | self._error_lines += 1 |
fc3fe1c2 | 985 | if better_err: |
4653a882 | 986 | Print('\n'.join(better_err), colour=self.col.GREEN) |
28370c1b | 987 | self._error_lines += 1 |
fc3fe1c2 | 988 | if worse_err: |
4653a882 | 989 | Print('\n'.join(worse_err), colour=self.col.RED) |
28370c1b | 990 | self._error_lines += 1 |
e30965db | 991 | if better_warn: |
4653a882 | 992 | Print('\n'.join(better_warn), colour=self.col.CYAN) |
e30965db SG |
993 | self._error_lines += 1 |
994 | if worse_warn: | |
4653a882 | 995 | Print('\n'.join(worse_warn), colour=self.col.MAGENTA) |
e30965db | 996 | self._error_lines += 1 |
fc3fe1c2 SG |
997 | |
998 | if show_sizes: | |
999 | self.PrintSizeSummary(board_selected, board_dict, show_detail, | |
1000 | show_bloat) | |
1001 | ||
1002 | # Save our updated information for the next call to this function | |
1003 | self._base_board_dict = board_dict | |
1004 | self._base_err_lines = err_lines | |
e30965db SG |
1005 | self._base_warn_lines = warn_lines |
1006 | self._base_err_line_boards = err_line_boards | |
1007 | self._base_warn_line_boards = warn_line_boards | |
fc3fe1c2 SG |
1008 | |
1009 | # Get a list of boards that did not get built, if needed | |
1010 | not_built = [] | |
1011 | for board in board_selected: | |
1012 | if not board in board_dict: | |
1013 | not_built.append(board) | |
1014 | if not_built: | |
4653a882 SG |
1015 | Print("Boards not built (%d): %s" % (len(not_built), |
1016 | ', '.join(not_built))) | |
fc3fe1c2 | 1017 | |
b2ea7ab2 | 1018 | def ProduceResultSummary(self, commit_upto, commits, board_selected): |
e30965db SG |
1019 | (board_dict, err_lines, err_line_boards, warn_lines, |
1020 | warn_line_boards) = self.GetResultSummary( | |
ed966657 SG |
1021 | board_selected, commit_upto, |
1022 | read_func_sizes=self._show_bloat) | |
b2ea7ab2 SG |
1023 | if commits: |
1024 | msg = '%02d: %s' % (commit_upto + 1, | |
1025 | commits[commit_upto].subject) | |
4653a882 | 1026 | Print(msg, colour=self.col.BLUE) |
b2ea7ab2 | 1027 | self.PrintResultSummary(board_selected, board_dict, |
ed966657 | 1028 | err_lines if self._show_errors else [], err_line_boards, |
e30965db | 1029 | warn_lines if self._show_errors else [], warn_line_boards, |
b2ea7ab2 | 1030 | self._show_sizes, self._show_detail, self._show_bloat) |
fc3fe1c2 | 1031 | |
b2ea7ab2 | 1032 | def ShowSummary(self, commits, board_selected): |
fc3fe1c2 SG |
1033 | """Show a build summary for U-Boot for a given board list. |
1034 | ||
1035 | Reset the result summary, then repeatedly call GetResultSummary on | |
1036 | each commit's results, then display the differences we see. | |
1037 | ||
1038 | Args: | |
1039 | commit: Commit objects to summarise | |
1040 | board_selected: Dict containing boards to summarise | |
fc3fe1c2 | 1041 | """ |
fea5858e | 1042 | self.commit_count = len(commits) if commits else 1 |
fc3fe1c2 SG |
1043 | self.commits = commits |
1044 | self.ResetResultSummary(board_selected) | |
28370c1b | 1045 | self._error_lines = 0 |
fc3fe1c2 SG |
1046 | |
1047 | for commit_upto in range(0, self.commit_count, self._step): | |
b2ea7ab2 | 1048 | self.ProduceResultSummary(commit_upto, commits, board_selected) |
28370c1b | 1049 | if not self._error_lines: |
4653a882 | 1050 | Print('(no errors to report)', colour=self.col.GREEN) |
fc3fe1c2 SG |
1051 | |
1052 | ||
1053 | def SetupBuild(self, board_selected, commits): | |
1054 | """Set up ready to start a build. | |
1055 | ||
1056 | Args: | |
1057 | board_selected: Selected boards to build | |
1058 | commits: Selected commits to build | |
1059 | """ | |
1060 | # First work out how many commits we will build | |
fea5858e | 1061 | count = (self.commit_count + self._step - 1) / self._step |
fc3fe1c2 SG |
1062 | self.count = len(board_selected) * count |
1063 | self.upto = self.warned = self.fail = 0 | |
1064 | self._timestamps = collections.deque() | |
1065 | ||
fc3fe1c2 SG |
1066 | def GetThreadDir(self, thread_num): |
1067 | """Get the directory path to the working dir for a thread. | |
1068 | ||
1069 | Args: | |
1070 | thread_num: Number of thread to check. | |
1071 | """ | |
1072 | return os.path.join(self._working_dir, '%02d' % thread_num) | |
1073 | ||
fea5858e | 1074 | def _PrepareThread(self, thread_num, setup_git): |
fc3fe1c2 SG |
1075 | """Prepare the working directory for a thread. |
1076 | ||
1077 | This clones or fetches the repo into the thread's work directory. | |
1078 | ||
1079 | Args: | |
1080 | thread_num: Thread number (0, 1, ...) | |
fea5858e | 1081 | setup_git: True to set up a git repo clone |
fc3fe1c2 SG |
1082 | """ |
1083 | thread_dir = self.GetThreadDir(thread_num) | |
190064b4 | 1084 | builderthread.Mkdir(thread_dir) |
fc3fe1c2 SG |
1085 | git_dir = os.path.join(thread_dir, '.git') |
1086 | ||
1087 | # Clone the repo if it doesn't already exist | |
1088 | # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so | |
1089 | # we have a private index but uses the origin repo's contents? | |
fea5858e | 1090 | if setup_git and self.git_dir: |
fc3fe1c2 SG |
1091 | src_dir = os.path.abspath(self.git_dir) |
1092 | if os.path.exists(git_dir): | |
1093 | gitutil.Fetch(git_dir, thread_dir) | |
1094 | else: | |
4653a882 | 1095 | Print('Cloning repo for thread %d' % thread_num) |
fc3fe1c2 SG |
1096 | gitutil.Clone(src_dir, thread_dir) |
1097 | ||
fea5858e | 1098 | def _PrepareWorkingSpace(self, max_threads, setup_git): |
fc3fe1c2 SG |
1099 | """Prepare the working directory for use. |
1100 | ||
1101 | Set up the git repo for each thread. | |
1102 | ||
1103 | Args: | |
1104 | max_threads: Maximum number of threads we expect to need. | |
fea5858e | 1105 | setup_git: True to set up a git repo clone |
fc3fe1c2 | 1106 | """ |
190064b4 | 1107 | builderthread.Mkdir(self._working_dir) |
fc3fe1c2 | 1108 | for thread in range(max_threads): |
fea5858e | 1109 | self._PrepareThread(thread, setup_git) |
fc3fe1c2 SG |
1110 | |
1111 | def _PrepareOutputSpace(self): | |
1112 | """Get the output directories ready to receive files. | |
1113 | ||
1114 | We delete any output directories which look like ones we need to | |
1115 | create. Having left over directories is confusing when the user wants | |
1116 | to check the output manually. | |
1117 | """ | |
1a915675 SG |
1118 | if not self.commits: |
1119 | return | |
fc3fe1c2 SG |
1120 | dir_list = [] |
1121 | for commit_upto in range(self.commit_count): | |
1122 | dir_list.append(self._GetOutputDir(commit_upto)) | |
1123 | ||
1124 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): | |
1125 | if dirname not in dir_list: | |
1126 | shutil.rmtree(dirname) | |
1127 | ||
e5a0e5d8 | 1128 | def BuildBoards(self, commits, board_selected, keep_outputs, verbose): |
fc3fe1c2 SG |
1129 | """Build all commits for a list of boards |
1130 | ||
1131 | Args: | |
1132 | commits: List of commits to be build, each a Commit object | |
1133 | boards_selected: Dict of selected boards, key is target name, | |
1134 | value is Board object | |
fc3fe1c2 | 1135 | keep_outputs: True to save build output files |
e5a0e5d8 | 1136 | verbose: Display build results as they are completed |
2c3deb97 SG |
1137 | Returns: |
1138 | Tuple containing: | |
1139 | - number of boards that failed to build | |
1140 | - number of boards that issued warnings | |
fc3fe1c2 | 1141 | """ |
fea5858e | 1142 | self.commit_count = len(commits) if commits else 1 |
fc3fe1c2 | 1143 | self.commits = commits |
e5a0e5d8 | 1144 | self._verbose = verbose |
fc3fe1c2 SG |
1145 | |
1146 | self.ResetResultSummary(board_selected) | |
f3d015cb | 1147 | builderthread.Mkdir(self.base_dir, parents = True) |
fea5858e SG |
1148 | self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), |
1149 | commits is not None) | |
fc3fe1c2 SG |
1150 | self._PrepareOutputSpace() |
1151 | self.SetupBuild(board_selected, commits) | |
1152 | self.ProcessResult(None) | |
1153 | ||
1154 | # Create jobs to build all commits for each board | |
1155 | for brd in board_selected.itervalues(): | |
190064b4 | 1156 | job = builderthread.BuilderJob() |
fc3fe1c2 SG |
1157 | job.board = brd |
1158 | job.commits = commits | |
1159 | job.keep_outputs = keep_outputs | |
1160 | job.step = self._step | |
1161 | self.queue.put(job) | |
1162 | ||
1163 | # Wait until all jobs are started | |
1164 | self.queue.join() | |
1165 | ||
1166 | # Wait until we have processed all output | |
1167 | self.out_queue.join() | |
4653a882 | 1168 | Print() |
fc3fe1c2 | 1169 | self.ClearLine(0) |
2c3deb97 | 1170 | return (self.fail, self.warned) |