]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
3 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> | |
4 | # | |
1a459660 | 5 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
6 | # |
7 | ||
8 | import collections | |
fc3fe1c2 SG |
9 | from datetime import datetime, timedelta |
10 | import glob | |
11 | import os | |
12 | import re | |
13 | import Queue | |
14 | import shutil | |
15 | import string | |
16 | import sys | |
fc3fe1c2 SG |
17 | import time |
18 | ||
190064b4 | 19 | import builderthread |
fc3fe1c2 SG |
20 | import command |
21 | import gitutil | |
22 | import terminal | |
4653a882 | 23 | from terminal import Print |
fc3fe1c2 SG |
24 | import toolchain |
25 | ||
26 | ||
27 | """ | |
28 | Theory of Operation | |
29 | ||
30 | Please see README for user documentation, and you should be familiar with | |
31 | that before trying to make sense of this. | |
32 | ||
33 | Buildman works by keeping the machine as busy as possible, building different | |
34 | commits for different boards on multiple CPUs at once. | |
35 | ||
36 | The source repo (self.git_dir) contains all the commits to be built. Each | |
37 | thread works on a single board at a time. It checks out the first commit, | |
38 | configures it for that board, then builds it. Then it checks out the next | |
39 | commit and builds it (typically without re-configuring). When it runs out | |
40 | of commits, it gets another job from the builder and starts again with that | |
41 | board. | |
42 | ||
43 | Clearly the builder threads could work either way - they could check out a | |
44 | commit and then built it for all boards. Using separate directories for each | |
45 | commit/board pair they could leave their build product around afterwards | |
46 | also. | |
47 | ||
48 | The intent behind building a single board for multiple commits, is to make | |
49 | use of incremental builds. Since each commit is built incrementally from | |
50 | the previous one, builds are faster. Reconfiguring for a different board | |
51 | removes all intermediate object files. | |
52 | ||
53 | Many threads can be working at once, but each has its own working directory. | |
54 | When a thread finishes a build, it puts the output files into a result | |
55 | directory. | |
56 | ||
57 | The base directory used by buildman is normally '../<branch>', i.e. | |
58 | a directory higher than the source repository and named after the branch | |
59 | being built. | |
60 | ||
61 | Within the base directory, we have one subdirectory for each commit. Within | |
62 | that is one subdirectory for each board. Within that is the build output for | |
63 | that commit/board combination. | |
64 | ||
65 | Buildman also create working directories for each thread, in a .bm-work/ | |
66 | subdirectory in the base dir. | |
67 | ||
68 | As an example, say we are building branch 'us-net' for boards 'sandbox' and | |
69 | 'seaboard', and say that us-net has two commits. We will have directories | |
70 | like this: | |
71 | ||
72 | us-net/ base directory | |
73 | 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ | |
74 | sandbox/ | |
75 | u-boot.bin | |
76 | seaboard/ | |
77 | u-boot.bin | |
78 | 02_of_02_g4ed4ebc_net--Check-tftp-comp/ | |
79 | sandbox/ | |
80 | u-boot.bin | |
81 | seaboard/ | |
82 | u-boot.bin | |
83 | .bm-work/ | |
84 | 00/ working directory for thread 0 (contains source checkout) | |
85 | build/ build output | |
86 | 01/ working directory for thread 1 | |
87 | build/ build output | |
88 | ... | |
89 | u-boot/ source directory | |
90 | .git/ repository | |
91 | """ | |
92 | ||
93 | # Possible build outcomes | |
94 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) | |
95 | ||
96 | # Translate a commit subject into a valid filename | |
97 | trans_valid_chars = string.maketrans("/: ", "---") | |
98 | ||
99 | ||
fc3fe1c2 SG |
100 | class Builder: |
101 | """Class for building U-Boot for a particular commit. | |
102 | ||
103 | Public members: (many should ->private) | |
104 | active: True if the builder is active and has not been stopped | |
105 | already_done: Number of builds already completed | |
106 | base_dir: Base directory to use for builder | |
107 | checkout: True to check out source, False to skip that step. | |
108 | This is used for testing. | |
109 | col: terminal.Color() object | |
110 | count: Number of commits to build | |
111 | do_make: Method to call to invoke Make | |
112 | fail: Number of builds that failed due to error | |
113 | force_build: Force building even if a build already exists | |
114 | force_config_on_failure: If a commit fails for a board, disable | |
115 | incremental building for the next commit we build for that | |
116 | board, so that we will see all warnings/errors again. | |
4266dc28 SG |
117 | force_build_failures: If a previously-built build (i.e. built on |
118 | a previous run of buildman) is marked as failed, rebuild it. | |
fc3fe1c2 SG |
119 | git_dir: Git directory containing source repository |
120 | last_line_len: Length of the last line we printed (used for erasing | |
121 | it with new progress information) | |
122 | num_jobs: Number of jobs to run at once (passed to make as -j) | |
123 | num_threads: Number of builder threads to run | |
124 | out_queue: Queue of results to process | |
125 | re_make_err: Compiled regular expression for ignore_lines | |
126 | queue: Queue of jobs to run | |
127 | threads: List of active threads | |
128 | toolchains: Toolchains object to use for building | |
129 | upto: Current commit number we are building (0.count-1) | |
130 | warned: Number of builds that produced at least one warning | |
97e91526 SG |
131 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
132 | incremental building, where buildman reconfigures on the first | |
133 | commit for a baord, and then just does an incremental build for | |
134 | the following commits. In fact buildman will reconfigure and | |
135 | retry for any failing commits, so generally the only effect of | |
136 | this option is to slow things down. | |
189a4968 SG |
137 | in_tree: Build U-Boot in-tree instead of specifying an output |
138 | directory separate from the source code. This option is really | |
139 | only useful for testing in-tree builds. | |
fc3fe1c2 SG |
140 | |
141 | Private members: | |
142 | _base_board_dict: Last-summarised Dict of boards | |
143 | _base_err_lines: Last-summarised list of errors | |
e30965db | 144 | _base_warn_lines: Last-summarised list of warnings |
fc3fe1c2 SG |
145 | _build_period_us: Time taken for a single build (float object). |
146 | _complete_delay: Expected delay until completion (timedelta) | |
147 | _next_delay_update: Next time we plan to display a progress update | |
148 | (datatime) | |
149 | _show_unknown: Show unknown boards (those not built) in summary | |
150 | _timestamps: List of timestamps for the completion of the last | |
151 | last _timestamp_count builds. Each is a datetime object. | |
152 | _timestamp_count: Number of timestamps to keep in our list. | |
153 | _working_dir: Base working directory containing all threads | |
154 | """ | |
155 | class Outcome: | |
156 | """Records a build outcome for a single make invocation | |
157 | ||
158 | Public Members: | |
159 | rc: Outcome value (OUTCOME_...) | |
160 | err_lines: List of error lines or [] if none | |
161 | sizes: Dictionary of image size information, keyed by filename | |
162 | - Each value is itself a dictionary containing | |
163 | values for 'text', 'data' and 'bss', being the integer | |
164 | size in bytes of each section. | |
165 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each | |
166 | value is itself a dictionary: | |
167 | key: function name | |
168 | value: Size of function in bytes | |
169 | """ | |
170 | def __init__(self, rc, err_lines, sizes, func_sizes): | |
171 | self.rc = rc | |
172 | self.err_lines = err_lines | |
173 | self.sizes = sizes | |
174 | self.func_sizes = func_sizes | |
175 | ||
176 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, | |
5971ab5c | 177 | gnu_make='make', checkout=True, show_unknown=True, step=1, |
bb1501f2 | 178 | no_subdirs=False, full_path=False): |
fc3fe1c2 SG |
179 | """Create a new Builder object |
180 | ||
181 | Args: | |
182 | toolchains: Toolchains object to use for building | |
183 | base_dir: Base directory to use for builder | |
184 | git_dir: Git directory containing source repository | |
185 | num_threads: Number of builder threads to run | |
186 | num_jobs: Number of jobs to run at once (passed to make as -j) | |
99796923 | 187 | gnu_make: the command name of GNU Make. |
fc3fe1c2 SG |
188 | checkout: True to check out source, False to skip that step. |
189 | This is used for testing. | |
190 | show_unknown: Show unknown boards (those not built) in summary | |
191 | step: 1 to process every commit, n to process every nth commit | |
bb1501f2 SG |
192 | no_subdirs: Don't create subdirectories when building current |
193 | source for a single board | |
194 | full_path: Return the full path in CROSS_COMPILE and don't set | |
195 | PATH | |
fc3fe1c2 SG |
196 | """ |
197 | self.toolchains = toolchains | |
198 | self.base_dir = base_dir | |
199 | self._working_dir = os.path.join(base_dir, '.bm-work') | |
200 | self.threads = [] | |
201 | self.active = True | |
202 | self.do_make = self.Make | |
99796923 | 203 | self.gnu_make = gnu_make |
fc3fe1c2 SG |
204 | self.checkout = checkout |
205 | self.num_threads = num_threads | |
206 | self.num_jobs = num_jobs | |
207 | self.already_done = 0 | |
208 | self.force_build = False | |
209 | self.git_dir = git_dir | |
210 | self._show_unknown = show_unknown | |
211 | self._timestamp_count = 10 | |
212 | self._build_period_us = None | |
213 | self._complete_delay = None | |
214 | self._next_delay_update = datetime.now() | |
215 | self.force_config_on_failure = True | |
4266dc28 | 216 | self.force_build_failures = False |
97e91526 | 217 | self.force_reconfig = False |
fc3fe1c2 | 218 | self._step = step |
189a4968 | 219 | self.in_tree = False |
28370c1b | 220 | self._error_lines = 0 |
5971ab5c | 221 | self.no_subdirs = no_subdirs |
bb1501f2 | 222 | self.full_path = full_path |
fc3fe1c2 SG |
223 | |
224 | self.col = terminal.Color() | |
225 | ||
e30965db SG |
226 | self._re_function = re.compile('(.*): In function.*') |
227 | self._re_files = re.compile('In file included from.*') | |
228 | self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') | |
229 | self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') | |
230 | ||
fc3fe1c2 SG |
231 | self.queue = Queue.Queue() |
232 | self.out_queue = Queue.Queue() | |
233 | for i in range(self.num_threads): | |
190064b4 | 234 | t = builderthread.BuilderThread(self, i) |
fc3fe1c2 SG |
235 | t.setDaemon(True) |
236 | t.start() | |
237 | self.threads.append(t) | |
238 | ||
239 | self.last_line_len = 0 | |
190064b4 | 240 | t = builderthread.ResultThread(self) |
fc3fe1c2 SG |
241 | t.setDaemon(True) |
242 | t.start() | |
243 | self.threads.append(t) | |
244 | ||
245 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] | |
246 | self.re_make_err = re.compile('|'.join(ignore_lines)) | |
247 | ||
248 | def __del__(self): | |
249 | """Get rid of all threads created by the builder""" | |
250 | for t in self.threads: | |
251 | del t | |
252 | ||
b2ea7ab2 | 253 | def SetDisplayOptions(self, show_errors=False, show_sizes=False, |
ed966657 SG |
254 | show_detail=False, show_bloat=False, |
255 | list_error_boards=False): | |
b2ea7ab2 SG |
256 | """Setup display options for the builder. |
257 | ||
258 | show_errors: True to show summarised error/warning info | |
259 | show_sizes: Show size deltas | |
260 | show_detail: Show detail for each board | |
261 | show_bloat: Show detail for each function | |
ed966657 | 262 | list_error_boards: Show the boards which caused each error/warning |
b2ea7ab2 SG |
263 | """ |
264 | self._show_errors = show_errors | |
265 | self._show_sizes = show_sizes | |
266 | self._show_detail = show_detail | |
267 | self._show_bloat = show_bloat | |
ed966657 | 268 | self._list_error_boards = list_error_boards |
b2ea7ab2 | 269 | |
fc3fe1c2 SG |
270 | def _AddTimestamp(self): |
271 | """Add a new timestamp to the list and record the build period. | |
272 | ||
273 | The build period is the length of time taken to perform a single | |
274 | build (one board, one commit). | |
275 | """ | |
276 | now = datetime.now() | |
277 | self._timestamps.append(now) | |
278 | count = len(self._timestamps) | |
279 | delta = self._timestamps[-1] - self._timestamps[0] | |
280 | seconds = delta.total_seconds() | |
281 | ||
282 | # If we have enough data, estimate build period (time taken for a | |
283 | # single build) and therefore completion time. | |
284 | if count > 1 and self._next_delay_update < now: | |
285 | self._next_delay_update = now + timedelta(seconds=2) | |
286 | if seconds > 0: | |
287 | self._build_period = float(seconds) / count | |
288 | todo = self.count - self.upto | |
289 | self._complete_delay = timedelta(microseconds= | |
290 | self._build_period * todo * 1000000) | |
291 | # Round it | |
292 | self._complete_delay -= timedelta( | |
293 | microseconds=self._complete_delay.microseconds) | |
294 | ||
295 | if seconds > 60: | |
296 | self._timestamps.popleft() | |
297 | count -= 1 | |
298 | ||
299 | def ClearLine(self, length): | |
300 | """Clear any characters on the current line | |
301 | ||
302 | Make way for a new line of length 'length', by outputting enough | |
303 | spaces to clear out the old line. Then remember the new length for | |
304 | next time. | |
305 | ||
306 | Args: | |
307 | length: Length of new line, in characters | |
308 | """ | |
309 | if length < self.last_line_len: | |
4653a882 SG |
310 | Print(' ' * (self.last_line_len - length), newline=False) |
311 | Print('\r', newline=False) | |
fc3fe1c2 SG |
312 | self.last_line_len = length |
313 | sys.stdout.flush() | |
314 | ||
315 | def SelectCommit(self, commit, checkout=True): | |
316 | """Checkout the selected commit for this build | |
317 | """ | |
318 | self.commit = commit | |
319 | if checkout and self.checkout: | |
320 | gitutil.Checkout(commit.hash) | |
321 | ||
322 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): | |
323 | """Run make | |
324 | ||
325 | Args: | |
326 | commit: Commit object that is being built | |
327 | brd: Board object that is being built | |
fd18a89e | 328 | stage: Stage that we are at (mrproper, config, build) |
fc3fe1c2 SG |
329 | cwd: Directory where make should be run |
330 | args: Arguments to pass to make | |
331 | kwargs: Arguments to pass to command.RunPipe() | |
332 | """ | |
99796923 | 333 | cmd = [self.gnu_make] + list(args) |
fc3fe1c2 SG |
334 | result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
335 | cwd=cwd, raise_on_error=False, **kwargs) | |
336 | return result | |
337 | ||
338 | def ProcessResult(self, result): | |
339 | """Process the result of a build, showing progress information | |
340 | ||
341 | Args: | |
e5a0e5d8 SG |
342 | result: A CommandResult object, which indicates the result for |
343 | a single build | |
fc3fe1c2 SG |
344 | """ |
345 | col = terminal.Color() | |
346 | if result: | |
347 | target = result.brd.target | |
348 | ||
349 | if result.return_code < 0: | |
350 | self.active = False | |
351 | command.StopAll() | |
352 | return | |
353 | ||
354 | self.upto += 1 | |
355 | if result.return_code != 0: | |
356 | self.fail += 1 | |
357 | elif result.stderr: | |
358 | self.warned += 1 | |
359 | if result.already_done: | |
360 | self.already_done += 1 | |
e5a0e5d8 | 361 | if self._verbose: |
4653a882 | 362 | Print('\r', newline=False) |
e5a0e5d8 SG |
363 | self.ClearLine(0) |
364 | boards_selected = {target : result.brd} | |
365 | self.ResetResultSummary(boards_selected) | |
366 | self.ProduceResultSummary(result.commit_upto, self.commits, | |
367 | boards_selected) | |
fc3fe1c2 SG |
368 | else: |
369 | target = '(starting)' | |
370 | ||
371 | # Display separate counts for ok, warned and fail | |
372 | ok = self.upto - self.warned - self.fail | |
373 | line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) | |
374 | line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) | |
375 | line += self.col.Color(self.col.RED, '%5d' % self.fail) | |
376 | ||
377 | name = ' /%-5d ' % self.count | |
378 | ||
379 | # Add our current completion time estimate | |
380 | self._AddTimestamp() | |
381 | if self._complete_delay: | |
382 | name += '%s : ' % self._complete_delay | |
383 | # When building all boards for a commit, we can print a commit | |
384 | # progress message. | |
385 | if result and result.commit_upto is None: | |
386 | name += 'commit %2d/%-3d' % (self.commit_upto + 1, | |
387 | self.commit_count) | |
388 | ||
389 | name += target | |
4653a882 | 390 | Print(line + name, newline=False) |
e5a0e5d8 | 391 | length = 14 + len(name) |
fc3fe1c2 SG |
392 | self.ClearLine(length) |
393 | ||
394 | def _GetOutputDir(self, commit_upto): | |
395 | """Get the name of the output directory for a commit number | |
396 | ||
397 | The output directory is typically .../<branch>/<commit>. | |
398 | ||
399 | Args: | |
400 | commit_upto: Commit number to use (0..self.count-1) | |
401 | """ | |
5971ab5c | 402 | commit_dir = None |
fea5858e SG |
403 | if self.commits: |
404 | commit = self.commits[commit_upto] | |
405 | subject = commit.subject.translate(trans_valid_chars) | |
406 | commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, | |
407 | self.commit_count, commit.hash, subject[:20])) | |
5971ab5c | 408 | elif not self.no_subdirs: |
fea5858e | 409 | commit_dir = 'current' |
5971ab5c SG |
410 | if not commit_dir: |
411 | return self.base_dir | |
412 | return os.path.join(self.base_dir, commit_dir) | |
fc3fe1c2 SG |
413 | |
414 | def GetBuildDir(self, commit_upto, target): | |
415 | """Get the name of the build directory for a commit number | |
416 | ||
417 | The build directory is typically .../<branch>/<commit>/<target>. | |
418 | ||
419 | Args: | |
420 | commit_upto: Commit number to use (0..self.count-1) | |
421 | target: Target name | |
422 | """ | |
423 | output_dir = self._GetOutputDir(commit_upto) | |
424 | return os.path.join(output_dir, target) | |
425 | ||
426 | def GetDoneFile(self, commit_upto, target): | |
427 | """Get the name of the done file for a commit number | |
428 | ||
429 | Args: | |
430 | commit_upto: Commit number to use (0..self.count-1) | |
431 | target: Target name | |
432 | """ | |
433 | return os.path.join(self.GetBuildDir(commit_upto, target), 'done') | |
434 | ||
435 | def GetSizesFile(self, commit_upto, target): | |
436 | """Get the name of the sizes file for a commit number | |
437 | ||
438 | Args: | |
439 | commit_upto: Commit number to use (0..self.count-1) | |
440 | target: Target name | |
441 | """ | |
442 | return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') | |
443 | ||
444 | def GetFuncSizesFile(self, commit_upto, target, elf_fname): | |
445 | """Get the name of the funcsizes file for a commit number and ELF file | |
446 | ||
447 | Args: | |
448 | commit_upto: Commit number to use (0..self.count-1) | |
449 | target: Target name | |
450 | elf_fname: Filename of elf image | |
451 | """ | |
452 | return os.path.join(self.GetBuildDir(commit_upto, target), | |
453 | '%s.sizes' % elf_fname.replace('/', '-')) | |
454 | ||
455 | def GetObjdumpFile(self, commit_upto, target, elf_fname): | |
456 | """Get the name of the objdump file for a commit number and ELF file | |
457 | ||
458 | Args: | |
459 | commit_upto: Commit number to use (0..self.count-1) | |
460 | target: Target name | |
461 | elf_fname: Filename of elf image | |
462 | """ | |
463 | return os.path.join(self.GetBuildDir(commit_upto, target), | |
464 | '%s.objdump' % elf_fname.replace('/', '-')) | |
465 | ||
466 | def GetErrFile(self, commit_upto, target): | |
467 | """Get the name of the err file for a commit number | |
468 | ||
469 | Args: | |
470 | commit_upto: Commit number to use (0..self.count-1) | |
471 | target: Target name | |
472 | """ | |
473 | output_dir = self.GetBuildDir(commit_upto, target) | |
474 | return os.path.join(output_dir, 'err') | |
475 | ||
476 | def FilterErrors(self, lines): | |
477 | """Filter out errors in which we have no interest | |
478 | ||
479 | We should probably use map(). | |
480 | ||
481 | Args: | |
482 | lines: List of error lines, each a string | |
483 | Returns: | |
484 | New list with only interesting lines included | |
485 | """ | |
486 | out_lines = [] | |
487 | for line in lines: | |
488 | if not self.re_make_err.search(line): | |
489 | out_lines.append(line) | |
490 | return out_lines | |
491 | ||
492 | def ReadFuncSizes(self, fname, fd): | |
493 | """Read function sizes from the output of 'nm' | |
494 | ||
495 | Args: | |
496 | fd: File containing data to read | |
497 | fname: Filename we are reading from (just for errors) | |
498 | ||
499 | Returns: | |
500 | Dictionary containing size of each function in bytes, indexed by | |
501 | function name. | |
502 | """ | |
503 | sym = {} | |
504 | for line in fd.readlines(): | |
505 | try: | |
506 | size, type, name = line[:-1].split() | |
507 | except: | |
4653a882 | 508 | Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) |
fc3fe1c2 SG |
509 | continue |
510 | if type in 'tTdDbB': | |
511 | # function names begin with '.' on 64-bit powerpc | |
512 | if '.' in name[1:]: | |
513 | name = 'static.' + name.split('.')[0] | |
514 | sym[name] = sym.get(name, 0) + int(size, 16) | |
515 | return sym | |
516 | ||
517 | def GetBuildOutcome(self, commit_upto, target, read_func_sizes): | |
518 | """Work out the outcome of a build. | |
519 | ||
520 | Args: | |
521 | commit_upto: Commit number to check (0..n-1) | |
522 | target: Target board to check | |
523 | read_func_sizes: True to read function size information | |
524 | ||
525 | Returns: | |
526 | Outcome object | |
527 | """ | |
528 | done_file = self.GetDoneFile(commit_upto, target) | |
529 | sizes_file = self.GetSizesFile(commit_upto, target) | |
530 | sizes = {} | |
531 | func_sizes = {} | |
532 | if os.path.exists(done_file): | |
533 | with open(done_file, 'r') as fd: | |
534 | return_code = int(fd.readline()) | |
535 | err_lines = [] | |
536 | err_file = self.GetErrFile(commit_upto, target) | |
537 | if os.path.exists(err_file): | |
538 | with open(err_file, 'r') as fd: | |
539 | err_lines = self.FilterErrors(fd.readlines()) | |
540 | ||
541 | # Decide whether the build was ok, failed or created warnings | |
542 | if return_code: | |
543 | rc = OUTCOME_ERROR | |
544 | elif len(err_lines): | |
545 | rc = OUTCOME_WARNING | |
546 | else: | |
547 | rc = OUTCOME_OK | |
548 | ||
549 | # Convert size information to our simple format | |
550 | if os.path.exists(sizes_file): | |
551 | with open(sizes_file, 'r') as fd: | |
552 | for line in fd.readlines(): | |
553 | values = line.split() | |
554 | rodata = 0 | |
555 | if len(values) > 6: | |
556 | rodata = int(values[6], 16) | |
557 | size_dict = { | |
558 | 'all' : int(values[0]) + int(values[1]) + | |
559 | int(values[2]), | |
560 | 'text' : int(values[0]) - rodata, | |
561 | 'data' : int(values[1]), | |
562 | 'bss' : int(values[2]), | |
563 | 'rodata' : rodata, | |
564 | } | |
565 | sizes[values[5]] = size_dict | |
566 | ||
567 | if read_func_sizes: | |
568 | pattern = self.GetFuncSizesFile(commit_upto, target, '*') | |
569 | for fname in glob.glob(pattern): | |
570 | with open(fname, 'r') as fd: | |
571 | dict_name = os.path.basename(fname).replace('.sizes', | |
572 | '') | |
573 | func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) | |
574 | ||
575 | return Builder.Outcome(rc, err_lines, sizes, func_sizes) | |
576 | ||
577 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) | |
578 | ||
579 | def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): | |
580 | """Calculate a summary of the results of building a commit. | |
581 | ||
582 | Args: | |
583 | board_selected: Dict containing boards to summarise | |
584 | commit_upto: Commit number to summarize (0..self.count-1) | |
585 | read_func_sizes: True to read function size information | |
586 | ||
587 | Returns: | |
588 | Tuple: | |
589 | Dict containing boards which passed building this commit. | |
590 | keyed by board.target | |
e30965db | 591 | List containing a summary of error lines |
ed966657 SG |
592 | Dict keyed by error line, containing a list of the Board |
593 | objects with that error | |
e30965db SG |
594 | List containing a summary of warning lines |
595 | Dict keyed by error line, containing a list of the Board | |
596 | objects with that warning | |
fc3fe1c2 | 597 | """ |
e30965db SG |
598 | def AddLine(lines_summary, lines_boards, line, board): |
599 | line = line.rstrip() | |
600 | if line in lines_boards: | |
601 | lines_boards[line].append(board) | |
602 | else: | |
603 | lines_boards[line] = [board] | |
604 | lines_summary.append(line) | |
605 | ||
fc3fe1c2 SG |
606 | board_dict = {} |
607 | err_lines_summary = [] | |
ed966657 | 608 | err_lines_boards = {} |
e30965db SG |
609 | warn_lines_summary = [] |
610 | warn_lines_boards = {} | |
fc3fe1c2 SG |
611 | |
612 | for board in boards_selected.itervalues(): | |
613 | outcome = self.GetBuildOutcome(commit_upto, board.target, | |
614 | read_func_sizes) | |
615 | board_dict[board.target] = outcome | |
e30965db SG |
616 | last_func = None |
617 | last_was_warning = False | |
618 | for line in outcome.err_lines: | |
619 | if line: | |
620 | if (self._re_function.match(line) or | |
621 | self._re_files.match(line)): | |
622 | last_func = line | |
ed966657 | 623 | else: |
e30965db SG |
624 | is_warning = self._re_warning.match(line) |
625 | is_note = self._re_note.match(line) | |
626 | if is_warning or (last_was_warning and is_note): | |
627 | if last_func: | |
628 | AddLine(warn_lines_summary, warn_lines_boards, | |
629 | last_func, board) | |
630 | AddLine(warn_lines_summary, warn_lines_boards, | |
631 | line, board) | |
632 | else: | |
633 | if last_func: | |
634 | AddLine(err_lines_summary, err_lines_boards, | |
635 | last_func, board) | |
636 | AddLine(err_lines_summary, err_lines_boards, | |
637 | line, board) | |
638 | last_was_warning = is_warning | |
639 | last_func = None | |
640 | return (board_dict, err_lines_summary, err_lines_boards, | |
641 | warn_lines_summary, warn_lines_boards) | |
fc3fe1c2 SG |
642 | |
643 | def AddOutcome(self, board_dict, arch_list, changes, char, color): | |
644 | """Add an output to our list of outcomes for each architecture | |
645 | ||
646 | This simple function adds failing boards (changes) to the | |
647 | relevant architecture string, so we can print the results out | |
648 | sorted by architecture. | |
649 | ||
650 | Args: | |
651 | board_dict: Dict containing all boards | |
652 | arch_list: Dict keyed by arch name. Value is a string containing | |
653 | a list of board names which failed for that arch. | |
654 | changes: List of boards to add to arch_list | |
655 | color: terminal.Colour object | |
656 | """ | |
657 | done_arch = {} | |
658 | for target in changes: | |
659 | if target in board_dict: | |
660 | arch = board_dict[target].arch | |
661 | else: | |
662 | arch = 'unknown' | |
663 | str = self.col.Color(color, ' ' + target) | |
664 | if not arch in done_arch: | |
665 | str = self.col.Color(color, char) + ' ' + str | |
666 | done_arch[arch] = True | |
667 | if not arch in arch_list: | |
668 | arch_list[arch] = str | |
669 | else: | |
670 | arch_list[arch] += str | |
671 | ||
672 | ||
673 | def ColourNum(self, num): | |
674 | color = self.col.RED if num > 0 else self.col.GREEN | |
675 | if num == 0: | |
676 | return '0' | |
677 | return self.col.Color(color, str(num)) | |
678 | ||
679 | def ResetResultSummary(self, board_selected): | |
680 | """Reset the results summary ready for use. | |
681 | ||
682 | Set up the base board list to be all those selected, and set the | |
683 | error lines to empty. | |
684 | ||
685 | Following this, calls to PrintResultSummary() will use this | |
686 | information to work out what has changed. | |
687 | ||
688 | Args: | |
689 | board_selected: Dict containing boards to summarise, keyed by | |
690 | board.target | |
691 | """ | |
692 | self._base_board_dict = {} | |
693 | for board in board_selected: | |
694 | self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) | |
695 | self._base_err_lines = [] | |
e30965db SG |
696 | self._base_warn_lines = [] |
697 | self._base_err_line_boards = {} | |
698 | self._base_warn_line_boards = {} | |
fc3fe1c2 SG |
699 | |
700 | def PrintFuncSizeDetail(self, fname, old, new): | |
701 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 | |
702 | delta, common = [], {} | |
703 | ||
704 | for a in old: | |
705 | if a in new: | |
706 | common[a] = 1 | |
707 | ||
708 | for name in old: | |
709 | if name not in common: | |
710 | remove += 1 | |
711 | down += old[name] | |
712 | delta.append([-old[name], name]) | |
713 | ||
714 | for name in new: | |
715 | if name not in common: | |
716 | add += 1 | |
717 | up += new[name] | |
718 | delta.append([new[name], name]) | |
719 | ||
720 | for name in common: | |
721 | diff = new.get(name, 0) - old.get(name, 0) | |
722 | if diff > 0: | |
723 | grow, up = grow + 1, up + diff | |
724 | elif diff < 0: | |
725 | shrink, down = shrink + 1, down - diff | |
726 | delta.append([diff, name]) | |
727 | ||
728 | delta.sort() | |
729 | delta.reverse() | |
730 | ||
731 | args = [add, -remove, grow, -shrink, up, -down, up - down] | |
732 | if max(args) == 0: | |
733 | return | |
734 | args = [self.ColourNum(x) for x in args] | |
735 | indent = ' ' * 15 | |
4653a882 SG |
736 | Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
737 | tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) | |
738 | Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', | |
739 | 'delta')) | |
fc3fe1c2 SG |
740 | for diff, name in delta: |
741 | if diff: | |
742 | color = self.col.RED if diff > 0 else self.col.GREEN | |
743 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, | |
744 | old.get(name, '-'), new.get(name,'-'), diff) | |
4653a882 | 745 | Print(msg, colour=color) |
fc3fe1c2 SG |
746 | |
747 | ||
748 | def PrintSizeDetail(self, target_list, show_bloat): | |
749 | """Show details size information for each board | |
750 | ||
751 | Args: | |
752 | target_list: List of targets, each a dict containing: | |
753 | 'target': Target name | |
754 | 'total_diff': Total difference in bytes across all areas | |
755 | <part_name>: Difference for that part | |
756 | show_bloat: Show detail for each function | |
757 | """ | |
758 | targets_by_diff = sorted(target_list, reverse=True, | |
759 | key=lambda x: x['_total_diff']) | |
760 | for result in targets_by_diff: | |
761 | printed_target = False | |
762 | for name in sorted(result): | |
763 | diff = result[name] | |
764 | if name.startswith('_'): | |
765 | continue | |
766 | if diff != 0: | |
767 | color = self.col.RED if diff > 0 else self.col.GREEN | |
768 | msg = ' %s %+d' % (name, diff) | |
769 | if not printed_target: | |
4653a882 SG |
770 | Print('%10s %-15s:' % ('', result['_target']), |
771 | newline=False) | |
fc3fe1c2 | 772 | printed_target = True |
4653a882 | 773 | Print(msg, colour=color, newline=False) |
fc3fe1c2 | 774 | if printed_target: |
4653a882 | 775 | Print() |
fc3fe1c2 SG |
776 | if show_bloat: |
777 | target = result['_target'] | |
778 | outcome = result['_outcome'] | |
779 | base_outcome = self._base_board_dict[target] | |
780 | for fname in outcome.func_sizes: | |
781 | self.PrintFuncSizeDetail(fname, | |
782 | base_outcome.func_sizes[fname], | |
783 | outcome.func_sizes[fname]) | |
784 | ||
785 | ||
786 | def PrintSizeSummary(self, board_selected, board_dict, show_detail, | |
787 | show_bloat): | |
788 | """Print a summary of image sizes broken down by section. | |
789 | ||
790 | The summary takes the form of one line per architecture. The | |
791 | line contains deltas for each of the sections (+ means the section | |
792 | got bigger, - means smaller). The nunmbers are the average number | |
793 | of bytes that a board in this section increased by. | |
794 | ||
795 | For example: | |
796 | powerpc: (622 boards) text -0.0 | |
797 | arm: (285 boards) text -0.0 | |
798 | nds32: (3 boards) text -8.0 | |
799 | ||
800 | Args: | |
801 | board_selected: Dict containing boards to summarise, keyed by | |
802 | board.target | |
803 | board_dict: Dict containing boards for which we built this | |
804 | commit, keyed by board.target. The value is an Outcome object. | |
805 | show_detail: Show detail for each board | |
806 | show_bloat: Show detail for each function | |
807 | """ | |
808 | arch_list = {} | |
809 | arch_count = {} | |
810 | ||
811 | # Calculate changes in size for different image parts | |
812 | # The previous sizes are in Board.sizes, for each board | |
813 | for target in board_dict: | |
814 | if target not in board_selected: | |
815 | continue | |
816 | base_sizes = self._base_board_dict[target].sizes | |
817 | outcome = board_dict[target] | |
818 | sizes = outcome.sizes | |
819 | ||
820 | # Loop through the list of images, creating a dict of size | |
821 | # changes for each image/part. We end up with something like | |
822 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} | |
823 | # which means that U-Boot data increased by 5 bytes and SPL | |
824 | # text decreased by 4. | |
825 | err = {'_target' : target} | |
826 | for image in sizes: | |
827 | if image in base_sizes: | |
828 | base_image = base_sizes[image] | |
829 | # Loop through the text, data, bss parts | |
830 | for part in sorted(sizes[image]): | |
831 | diff = sizes[image][part] - base_image[part] | |
832 | col = None | |
833 | if diff: | |
834 | if image == 'u-boot': | |
835 | name = part | |
836 | else: | |
837 | name = image + ':' + part | |
838 | err[name] = diff | |
839 | arch = board_selected[target].arch | |
840 | if not arch in arch_count: | |
841 | arch_count[arch] = 1 | |
842 | else: | |
843 | arch_count[arch] += 1 | |
844 | if not sizes: | |
845 | pass # Only add to our list when we have some stats | |
846 | elif not arch in arch_list: | |
847 | arch_list[arch] = [err] | |
848 | else: | |
849 | arch_list[arch].append(err) | |
850 | ||
851 | # We now have a list of image size changes sorted by arch | |
852 | # Print out a summary of these | |
853 | for arch, target_list in arch_list.iteritems(): | |
854 | # Get total difference for each type | |
855 | totals = {} | |
856 | for result in target_list: | |
857 | total = 0 | |
858 | for name, diff in result.iteritems(): | |
859 | if name.startswith('_'): | |
860 | continue | |
861 | total += diff | |
862 | if name in totals: | |
863 | totals[name] += diff | |
864 | else: | |
865 | totals[name] = diff | |
866 | result['_total_diff'] = total | |
867 | result['_outcome'] = board_dict[result['_target']] | |
868 | ||
869 | count = len(target_list) | |
870 | printed_arch = False | |
871 | for name in sorted(totals): | |
872 | diff = totals[name] | |
873 | if diff: | |
874 | # Display the average difference in this name for this | |
875 | # architecture | |
876 | avg_diff = float(diff) / count | |
877 | color = self.col.RED if avg_diff > 0 else self.col.GREEN | |
878 | msg = ' %s %+1.1f' % (name, avg_diff) | |
879 | if not printed_arch: | |
4653a882 SG |
880 | Print('%10s: (for %d/%d boards)' % (arch, count, |
881 | arch_count[arch]), newline=False) | |
fc3fe1c2 | 882 | printed_arch = True |
4653a882 | 883 | Print(msg, colour=color, newline=False) |
fc3fe1c2 SG |
884 | |
885 | if printed_arch: | |
4653a882 | 886 | Print() |
fc3fe1c2 SG |
887 | if show_detail: |
888 | self.PrintSizeDetail(target_list, show_bloat) | |
889 | ||
890 | ||
891 | def PrintResultSummary(self, board_selected, board_dict, err_lines, | |
e30965db SG |
892 | err_line_boards, warn_lines, warn_line_boards, |
893 | show_sizes, show_detail, show_bloat): | |
fc3fe1c2 SG |
894 | """Compare results with the base results and display delta. |
895 | ||
896 | Only boards mentioned in board_selected will be considered. This | |
897 | function is intended to be called repeatedly with the results of | |
898 | each commit. It therefore shows a 'diff' between what it saw in | |
899 | the last call and what it sees now. | |
900 | ||
901 | Args: | |
902 | board_selected: Dict containing boards to summarise, keyed by | |
903 | board.target | |
904 | board_dict: Dict containing boards for which we built this | |
905 | commit, keyed by board.target. The value is an Outcome object. | |
906 | err_lines: A list of errors for this commit, or [] if there is | |
907 | none, or we don't want to print errors | |
ed966657 SG |
908 | err_line_boards: Dict keyed by error line, containing a list of |
909 | the Board objects with that error | |
e30965db SG |
910 | warn_lines: A list of warnings for this commit, or [] if there is |
911 | none, or we don't want to print errors | |
912 | warn_line_boards: Dict keyed by warning line, containing a list of | |
913 | the Board objects with that warning | |
fc3fe1c2 SG |
914 | show_sizes: Show image size deltas |
915 | show_detail: Show detail for each board | |
916 | show_bloat: Show detail for each function | |
917 | """ | |
e30965db | 918 | def _BoardList(line, line_boards): |
ed966657 SG |
919 | """Helper function to get a line of boards containing a line |
920 | ||
921 | Args: | |
922 | line: Error line to search for | |
923 | Return: | |
924 | String containing a list of boards with that error line, or | |
925 | '' if the user has not requested such a list | |
926 | """ | |
927 | if self._list_error_boards: | |
928 | names = [] | |
e30965db | 929 | for board in line_boards[line]: |
f66153be SG |
930 | if not board.target in names: |
931 | names.append(board.target) | |
ed966657 SG |
932 | names_str = '(%s) ' % ','.join(names) |
933 | else: | |
934 | names_str = '' | |
935 | return names_str | |
936 | ||
e30965db SG |
937 | def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, |
938 | char): | |
939 | better_lines = [] | |
940 | worse_lines = [] | |
941 | for line in lines: | |
942 | if line not in base_lines: | |
943 | worse_lines.append(char + '+' + | |
944 | _BoardList(line, line_boards) + line) | |
945 | for line in base_lines: | |
946 | if line not in lines: | |
947 | better_lines.append(char + '-' + | |
948 | _BoardList(line, base_line_boards) + line) | |
949 | return better_lines, worse_lines | |
950 | ||
fc3fe1c2 SG |
951 | better = [] # List of boards fixed since last commit |
952 | worse = [] # List of new broken boards since last commit | |
953 | new = [] # List of boards that didn't exist last time | |
954 | unknown = [] # List of boards that were not built | |
955 | ||
956 | for target in board_dict: | |
957 | if target not in board_selected: | |
958 | continue | |
959 | ||
960 | # If the board was built last time, add its outcome to a list | |
961 | if target in self._base_board_dict: | |
962 | base_outcome = self._base_board_dict[target].rc | |
963 | outcome = board_dict[target] | |
964 | if outcome.rc == OUTCOME_UNKNOWN: | |
965 | unknown.append(target) | |
966 | elif outcome.rc < base_outcome: | |
967 | better.append(target) | |
968 | elif outcome.rc > base_outcome: | |
969 | worse.append(target) | |
970 | else: | |
971 | new.append(target) | |
972 | ||
973 | # Get a list of errors that have appeared, and disappeared | |
e30965db SG |
974 | better_err, worse_err = _CalcErrorDelta(self._base_err_lines, |
975 | self._base_err_line_boards, err_lines, err_line_boards, '') | |
976 | better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, | |
977 | self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') | |
fc3fe1c2 SG |
978 | |
979 | # Display results by arch | |
e30965db SG |
980 | if (better or worse or unknown or new or worse_err or better_err |
981 | or worse_warn or better_warn): | |
fc3fe1c2 SG |
982 | arch_list = {} |
983 | self.AddOutcome(board_selected, arch_list, better, '', | |
984 | self.col.GREEN) | |
985 | self.AddOutcome(board_selected, arch_list, worse, '+', | |
986 | self.col.RED) | |
987 | self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) | |
988 | if self._show_unknown: | |
989 | self.AddOutcome(board_selected, arch_list, unknown, '?', | |
990 | self.col.MAGENTA) | |
991 | for arch, target_list in arch_list.iteritems(): | |
4653a882 | 992 | Print('%10s: %s' % (arch, target_list)) |
28370c1b | 993 | self._error_lines += 1 |
fc3fe1c2 | 994 | if better_err: |
4653a882 | 995 | Print('\n'.join(better_err), colour=self.col.GREEN) |
28370c1b | 996 | self._error_lines += 1 |
fc3fe1c2 | 997 | if worse_err: |
4653a882 | 998 | Print('\n'.join(worse_err), colour=self.col.RED) |
28370c1b | 999 | self._error_lines += 1 |
e30965db | 1000 | if better_warn: |
4653a882 | 1001 | Print('\n'.join(better_warn), colour=self.col.CYAN) |
e30965db SG |
1002 | self._error_lines += 1 |
1003 | if worse_warn: | |
4653a882 | 1004 | Print('\n'.join(worse_warn), colour=self.col.MAGENTA) |
e30965db | 1005 | self._error_lines += 1 |
fc3fe1c2 SG |
1006 | |
1007 | if show_sizes: | |
1008 | self.PrintSizeSummary(board_selected, board_dict, show_detail, | |
1009 | show_bloat) | |
1010 | ||
1011 | # Save our updated information for the next call to this function | |
1012 | self._base_board_dict = board_dict | |
1013 | self._base_err_lines = err_lines | |
e30965db SG |
1014 | self._base_warn_lines = warn_lines |
1015 | self._base_err_line_boards = err_line_boards | |
1016 | self._base_warn_line_boards = warn_line_boards | |
fc3fe1c2 SG |
1017 | |
1018 | # Get a list of boards that did not get built, if needed | |
1019 | not_built = [] | |
1020 | for board in board_selected: | |
1021 | if not board in board_dict: | |
1022 | not_built.append(board) | |
1023 | if not_built: | |
4653a882 SG |
1024 | Print("Boards not built (%d): %s" % (len(not_built), |
1025 | ', '.join(not_built))) | |
fc3fe1c2 | 1026 | |
b2ea7ab2 | 1027 | def ProduceResultSummary(self, commit_upto, commits, board_selected): |
e30965db SG |
1028 | (board_dict, err_lines, err_line_boards, warn_lines, |
1029 | warn_line_boards) = self.GetResultSummary( | |
ed966657 SG |
1030 | board_selected, commit_upto, |
1031 | read_func_sizes=self._show_bloat) | |
b2ea7ab2 SG |
1032 | if commits: |
1033 | msg = '%02d: %s' % (commit_upto + 1, | |
1034 | commits[commit_upto].subject) | |
4653a882 | 1035 | Print(msg, colour=self.col.BLUE) |
b2ea7ab2 | 1036 | self.PrintResultSummary(board_selected, board_dict, |
ed966657 | 1037 | err_lines if self._show_errors else [], err_line_boards, |
e30965db | 1038 | warn_lines if self._show_errors else [], warn_line_boards, |
b2ea7ab2 | 1039 | self._show_sizes, self._show_detail, self._show_bloat) |
fc3fe1c2 | 1040 | |
b2ea7ab2 | 1041 | def ShowSummary(self, commits, board_selected): |
fc3fe1c2 SG |
1042 | """Show a build summary for U-Boot for a given board list. |
1043 | ||
1044 | Reset the result summary, then repeatedly call GetResultSummary on | |
1045 | each commit's results, then display the differences we see. | |
1046 | ||
1047 | Args: | |
1048 | commit: Commit objects to summarise | |
1049 | board_selected: Dict containing boards to summarise | |
fc3fe1c2 | 1050 | """ |
fea5858e | 1051 | self.commit_count = len(commits) if commits else 1 |
fc3fe1c2 SG |
1052 | self.commits = commits |
1053 | self.ResetResultSummary(board_selected) | |
28370c1b | 1054 | self._error_lines = 0 |
fc3fe1c2 SG |
1055 | |
1056 | for commit_upto in range(0, self.commit_count, self._step): | |
b2ea7ab2 | 1057 | self.ProduceResultSummary(commit_upto, commits, board_selected) |
28370c1b | 1058 | if not self._error_lines: |
4653a882 | 1059 | Print('(no errors to report)', colour=self.col.GREEN) |
fc3fe1c2 SG |
1060 | |
1061 | ||
1062 | def SetupBuild(self, board_selected, commits): | |
1063 | """Set up ready to start a build. | |
1064 | ||
1065 | Args: | |
1066 | board_selected: Selected boards to build | |
1067 | commits: Selected commits to build | |
1068 | """ | |
1069 | # First work out how many commits we will build | |
fea5858e | 1070 | count = (self.commit_count + self._step - 1) / self._step |
fc3fe1c2 SG |
1071 | self.count = len(board_selected) * count |
1072 | self.upto = self.warned = self.fail = 0 | |
1073 | self._timestamps = collections.deque() | |
1074 | ||
fc3fe1c2 SG |
1075 | def GetThreadDir(self, thread_num): |
1076 | """Get the directory path to the working dir for a thread. | |
1077 | ||
1078 | Args: | |
1079 | thread_num: Number of thread to check. | |
1080 | """ | |
1081 | return os.path.join(self._working_dir, '%02d' % thread_num) | |
1082 | ||
fea5858e | 1083 | def _PrepareThread(self, thread_num, setup_git): |
fc3fe1c2 SG |
1084 | """Prepare the working directory for a thread. |
1085 | ||
1086 | This clones or fetches the repo into the thread's work directory. | |
1087 | ||
1088 | Args: | |
1089 | thread_num: Thread number (0, 1, ...) | |
fea5858e | 1090 | setup_git: True to set up a git repo clone |
fc3fe1c2 SG |
1091 | """ |
1092 | thread_dir = self.GetThreadDir(thread_num) | |
190064b4 | 1093 | builderthread.Mkdir(thread_dir) |
fc3fe1c2 SG |
1094 | git_dir = os.path.join(thread_dir, '.git') |
1095 | ||
1096 | # Clone the repo if it doesn't already exist | |
1097 | # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so | |
1098 | # we have a private index but uses the origin repo's contents? | |
fea5858e | 1099 | if setup_git and self.git_dir: |
fc3fe1c2 SG |
1100 | src_dir = os.path.abspath(self.git_dir) |
1101 | if os.path.exists(git_dir): | |
1102 | gitutil.Fetch(git_dir, thread_dir) | |
1103 | else: | |
4653a882 | 1104 | Print('Cloning repo for thread %d' % thread_num) |
fc3fe1c2 SG |
1105 | gitutil.Clone(src_dir, thread_dir) |
1106 | ||
fea5858e | 1107 | def _PrepareWorkingSpace(self, max_threads, setup_git): |
fc3fe1c2 SG |
1108 | """Prepare the working directory for use. |
1109 | ||
1110 | Set up the git repo for each thread. | |
1111 | ||
1112 | Args: | |
1113 | max_threads: Maximum number of threads we expect to need. | |
fea5858e | 1114 | setup_git: True to set up a git repo clone |
fc3fe1c2 | 1115 | """ |
190064b4 | 1116 | builderthread.Mkdir(self._working_dir) |
fc3fe1c2 | 1117 | for thread in range(max_threads): |
fea5858e | 1118 | self._PrepareThread(thread, setup_git) |
fc3fe1c2 SG |
1119 | |
1120 | def _PrepareOutputSpace(self): | |
1121 | """Get the output directories ready to receive files. | |
1122 | ||
1123 | We delete any output directories which look like ones we need to | |
1124 | create. Having left over directories is confusing when the user wants | |
1125 | to check the output manually. | |
1126 | """ | |
1a915675 SG |
1127 | if not self.commits: |
1128 | return | |
fc3fe1c2 SG |
1129 | dir_list = [] |
1130 | for commit_upto in range(self.commit_count): | |
1131 | dir_list.append(self._GetOutputDir(commit_upto)) | |
1132 | ||
1133 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): | |
1134 | if dirname not in dir_list: | |
1135 | shutil.rmtree(dirname) | |
1136 | ||
e5a0e5d8 | 1137 | def BuildBoards(self, commits, board_selected, keep_outputs, verbose): |
fc3fe1c2 SG |
1138 | """Build all commits for a list of boards |
1139 | ||
1140 | Args: | |
1141 | commits: List of commits to be build, each a Commit object | |
1142 | boards_selected: Dict of selected boards, key is target name, | |
1143 | value is Board object | |
fc3fe1c2 | 1144 | keep_outputs: True to save build output files |
e5a0e5d8 | 1145 | verbose: Display build results as they are completed |
2c3deb97 SG |
1146 | Returns: |
1147 | Tuple containing: | |
1148 | - number of boards that failed to build | |
1149 | - number of boards that issued warnings | |
fc3fe1c2 | 1150 | """ |
fea5858e | 1151 | self.commit_count = len(commits) if commits else 1 |
fc3fe1c2 | 1152 | self.commits = commits |
e5a0e5d8 | 1153 | self._verbose = verbose |
fc3fe1c2 SG |
1154 | |
1155 | self.ResetResultSummary(board_selected) | |
f3d015cb | 1156 | builderthread.Mkdir(self.base_dir, parents = True) |
fea5858e SG |
1157 | self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), |
1158 | commits is not None) | |
fc3fe1c2 SG |
1159 | self._PrepareOutputSpace() |
1160 | self.SetupBuild(board_selected, commits) | |
1161 | self.ProcessResult(None) | |
1162 | ||
1163 | # Create jobs to build all commits for each board | |
1164 | for brd in board_selected.itervalues(): | |
190064b4 | 1165 | job = builderthread.BuilderJob() |
fc3fe1c2 SG |
1166 | job.board = brd |
1167 | job.commits = commits | |
1168 | job.keep_outputs = keep_outputs | |
1169 | job.step = self._step | |
1170 | self.queue.put(job) | |
1171 | ||
1172 | # Wait until all jobs are started | |
1173 | self.queue.join() | |
1174 | ||
1175 | # Wait until we have processed all output | |
1176 | self.out_queue.join() | |
4653a882 | 1177 | Print() |
fc3fe1c2 | 1178 | self.ClearLine(0) |
2c3deb97 | 1179 | return (self.fail, self.warned) |