]>
Commit | Line | Data |
---|---|---|
fc3fe1c2 SG |
1 | # Copyright (c) 2013 The Chromium OS Authors. |
2 | # | |
3 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> | |
4 | # | |
1a459660 | 5 | # SPDX-License-Identifier: GPL-2.0+ |
fc3fe1c2 SG |
6 | # |
7 | ||
8 | import collections | |
fc3fe1c2 SG |
9 | from datetime import datetime, timedelta |
10 | import glob | |
11 | import os | |
12 | import re | |
13 | import Queue | |
14 | import shutil | |
15 | import string | |
16 | import sys | |
fc3fe1c2 SG |
17 | import time |
18 | ||
190064b4 | 19 | import builderthread |
fc3fe1c2 SG |
20 | import command |
21 | import gitutil | |
22 | import terminal | |
23 | import toolchain | |
24 | ||
25 | ||
26 | """ | |
27 | Theory of Operation | |
28 | ||
29 | Please see README for user documentation, and you should be familiar with | |
30 | that before trying to make sense of this. | |
31 | ||
32 | Buildman works by keeping the machine as busy as possible, building different | |
33 | commits for different boards on multiple CPUs at once. | |
34 | ||
35 | The source repo (self.git_dir) contains all the commits to be built. Each | |
36 | thread works on a single board at a time. It checks out the first commit, | |
37 | configures it for that board, then builds it. Then it checks out the next | |
38 | commit and builds it (typically without re-configuring). When it runs out | |
39 | of commits, it gets another job from the builder and starts again with that | |
40 | board. | |
41 | ||
42 | Clearly the builder threads could work either way - they could check out a | |
43 | commit and then built it for all boards. Using separate directories for each | |
44 | commit/board pair they could leave their build product around afterwards | |
45 | also. | |
46 | ||
47 | The intent behind building a single board for multiple commits, is to make | |
48 | use of incremental builds. Since each commit is built incrementally from | |
49 | the previous one, builds are faster. Reconfiguring for a different board | |
50 | removes all intermediate object files. | |
51 | ||
52 | Many threads can be working at once, but each has its own working directory. | |
53 | When a thread finishes a build, it puts the output files into a result | |
54 | directory. | |
55 | ||
56 | The base directory used by buildman is normally '../<branch>', i.e. | |
57 | a directory higher than the source repository and named after the branch | |
58 | being built. | |
59 | ||
60 | Within the base directory, we have one subdirectory for each commit. Within | |
61 | that is one subdirectory for each board. Within that is the build output for | |
62 | that commit/board combination. | |
63 | ||
64 | Buildman also create working directories for each thread, in a .bm-work/ | |
65 | subdirectory in the base dir. | |
66 | ||
67 | As an example, say we are building branch 'us-net' for boards 'sandbox' and | |
68 | 'seaboard', and say that us-net has two commits. We will have directories | |
69 | like this: | |
70 | ||
71 | us-net/ base directory | |
72 | 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ | |
73 | sandbox/ | |
74 | u-boot.bin | |
75 | seaboard/ | |
76 | u-boot.bin | |
77 | 02_of_02_g4ed4ebc_net--Check-tftp-comp/ | |
78 | sandbox/ | |
79 | u-boot.bin | |
80 | seaboard/ | |
81 | u-boot.bin | |
82 | .bm-work/ | |
83 | 00/ working directory for thread 0 (contains source checkout) | |
84 | build/ build output | |
85 | 01/ working directory for thread 1 | |
86 | build/ build output | |
87 | ... | |
88 | u-boot/ source directory | |
89 | .git/ repository | |
90 | """ | |
91 | ||
92 | # Possible build outcomes | |
93 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) | |
94 | ||
95 | # Translate a commit subject into a valid filename | |
96 | trans_valid_chars = string.maketrans("/: ", "---") | |
97 | ||
98 | ||
fc3fe1c2 SG |
99 | class Builder: |
100 | """Class for building U-Boot for a particular commit. | |
101 | ||
102 | Public members: (many should ->private) | |
103 | active: True if the builder is active and has not been stopped | |
104 | already_done: Number of builds already completed | |
105 | base_dir: Base directory to use for builder | |
106 | checkout: True to check out source, False to skip that step. | |
107 | This is used for testing. | |
108 | col: terminal.Color() object | |
109 | count: Number of commits to build | |
110 | do_make: Method to call to invoke Make | |
111 | fail: Number of builds that failed due to error | |
112 | force_build: Force building even if a build already exists | |
113 | force_config_on_failure: If a commit fails for a board, disable | |
114 | incremental building for the next commit we build for that | |
115 | board, so that we will see all warnings/errors again. | |
4266dc28 SG |
116 | force_build_failures: If a previously-built build (i.e. built on |
117 | a previous run of buildman) is marked as failed, rebuild it. | |
fc3fe1c2 SG |
118 | git_dir: Git directory containing source repository |
119 | last_line_len: Length of the last line we printed (used for erasing | |
120 | it with new progress information) | |
121 | num_jobs: Number of jobs to run at once (passed to make as -j) | |
122 | num_threads: Number of builder threads to run | |
123 | out_queue: Queue of results to process | |
124 | re_make_err: Compiled regular expression for ignore_lines | |
125 | queue: Queue of jobs to run | |
126 | threads: List of active threads | |
127 | toolchains: Toolchains object to use for building | |
128 | upto: Current commit number we are building (0.count-1) | |
129 | warned: Number of builds that produced at least one warning | |
97e91526 SG |
130 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
131 | incremental building, where buildman reconfigures on the first | |
132 | commit for a baord, and then just does an incremental build for | |
133 | the following commits. In fact buildman will reconfigure and | |
134 | retry for any failing commits, so generally the only effect of | |
135 | this option is to slow things down. | |
189a4968 SG |
136 | in_tree: Build U-Boot in-tree instead of specifying an output |
137 | directory separate from the source code. This option is really | |
138 | only useful for testing in-tree builds. | |
fc3fe1c2 SG |
139 | |
140 | Private members: | |
141 | _base_board_dict: Last-summarised Dict of boards | |
142 | _base_err_lines: Last-summarised list of errors | |
143 | _build_period_us: Time taken for a single build (float object). | |
144 | _complete_delay: Expected delay until completion (timedelta) | |
145 | _next_delay_update: Next time we plan to display a progress update | |
146 | (datatime) | |
147 | _show_unknown: Show unknown boards (those not built) in summary | |
148 | _timestamps: List of timestamps for the completion of the last | |
149 | last _timestamp_count builds. Each is a datetime object. | |
150 | _timestamp_count: Number of timestamps to keep in our list. | |
151 | _working_dir: Base working directory containing all threads | |
152 | """ | |
153 | class Outcome: | |
154 | """Records a build outcome for a single make invocation | |
155 | ||
156 | Public Members: | |
157 | rc: Outcome value (OUTCOME_...) | |
158 | err_lines: List of error lines or [] if none | |
159 | sizes: Dictionary of image size information, keyed by filename | |
160 | - Each value is itself a dictionary containing | |
161 | values for 'text', 'data' and 'bss', being the integer | |
162 | size in bytes of each section. | |
163 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each | |
164 | value is itself a dictionary: | |
165 | key: function name | |
166 | value: Size of function in bytes | |
167 | """ | |
168 | def __init__(self, rc, err_lines, sizes, func_sizes): | |
169 | self.rc = rc | |
170 | self.err_lines = err_lines | |
171 | self.sizes = sizes | |
172 | self.func_sizes = func_sizes | |
173 | ||
174 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, | |
99796923 | 175 | gnu_make='make', checkout=True, show_unknown=True, step=1): |
fc3fe1c2 SG |
176 | """Create a new Builder object |
177 | ||
178 | Args: | |
179 | toolchains: Toolchains object to use for building | |
180 | base_dir: Base directory to use for builder | |
181 | git_dir: Git directory containing source repository | |
182 | num_threads: Number of builder threads to run | |
183 | num_jobs: Number of jobs to run at once (passed to make as -j) | |
99796923 | 184 | gnu_make: the command name of GNU Make. |
fc3fe1c2 SG |
185 | checkout: True to check out source, False to skip that step. |
186 | This is used for testing. | |
187 | show_unknown: Show unknown boards (those not built) in summary | |
188 | step: 1 to process every commit, n to process every nth commit | |
189 | """ | |
190 | self.toolchains = toolchains | |
191 | self.base_dir = base_dir | |
192 | self._working_dir = os.path.join(base_dir, '.bm-work') | |
193 | self.threads = [] | |
194 | self.active = True | |
195 | self.do_make = self.Make | |
99796923 | 196 | self.gnu_make = gnu_make |
fc3fe1c2 SG |
197 | self.checkout = checkout |
198 | self.num_threads = num_threads | |
199 | self.num_jobs = num_jobs | |
200 | self.already_done = 0 | |
201 | self.force_build = False | |
202 | self.git_dir = git_dir | |
203 | self._show_unknown = show_unknown | |
204 | self._timestamp_count = 10 | |
205 | self._build_period_us = None | |
206 | self._complete_delay = None | |
207 | self._next_delay_update = datetime.now() | |
208 | self.force_config_on_failure = True | |
4266dc28 | 209 | self.force_build_failures = False |
97e91526 | 210 | self.force_reconfig = False |
fc3fe1c2 | 211 | self._step = step |
189a4968 | 212 | self.in_tree = False |
28370c1b | 213 | self._error_lines = 0 |
fc3fe1c2 SG |
214 | |
215 | self.col = terminal.Color() | |
216 | ||
217 | self.queue = Queue.Queue() | |
218 | self.out_queue = Queue.Queue() | |
219 | for i in range(self.num_threads): | |
190064b4 | 220 | t = builderthread.BuilderThread(self, i) |
fc3fe1c2 SG |
221 | t.setDaemon(True) |
222 | t.start() | |
223 | self.threads.append(t) | |
224 | ||
225 | self.last_line_len = 0 | |
190064b4 | 226 | t = builderthread.ResultThread(self) |
fc3fe1c2 SG |
227 | t.setDaemon(True) |
228 | t.start() | |
229 | self.threads.append(t) | |
230 | ||
231 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] | |
232 | self.re_make_err = re.compile('|'.join(ignore_lines)) | |
233 | ||
234 | def __del__(self): | |
235 | """Get rid of all threads created by the builder""" | |
236 | for t in self.threads: | |
237 | del t | |
238 | ||
b2ea7ab2 SG |
239 | def SetDisplayOptions(self, show_errors=False, show_sizes=False, |
240 | show_detail=False, show_bloat=False): | |
241 | """Setup display options for the builder. | |
242 | ||
243 | show_errors: True to show summarised error/warning info | |
244 | show_sizes: Show size deltas | |
245 | show_detail: Show detail for each board | |
246 | show_bloat: Show detail for each function | |
247 | """ | |
248 | self._show_errors = show_errors | |
249 | self._show_sizes = show_sizes | |
250 | self._show_detail = show_detail | |
251 | self._show_bloat = show_bloat | |
252 | ||
fc3fe1c2 SG |
253 | def _AddTimestamp(self): |
254 | """Add a new timestamp to the list and record the build period. | |
255 | ||
256 | The build period is the length of time taken to perform a single | |
257 | build (one board, one commit). | |
258 | """ | |
259 | now = datetime.now() | |
260 | self._timestamps.append(now) | |
261 | count = len(self._timestamps) | |
262 | delta = self._timestamps[-1] - self._timestamps[0] | |
263 | seconds = delta.total_seconds() | |
264 | ||
265 | # If we have enough data, estimate build period (time taken for a | |
266 | # single build) and therefore completion time. | |
267 | if count > 1 and self._next_delay_update < now: | |
268 | self._next_delay_update = now + timedelta(seconds=2) | |
269 | if seconds > 0: | |
270 | self._build_period = float(seconds) / count | |
271 | todo = self.count - self.upto | |
272 | self._complete_delay = timedelta(microseconds= | |
273 | self._build_period * todo * 1000000) | |
274 | # Round it | |
275 | self._complete_delay -= timedelta( | |
276 | microseconds=self._complete_delay.microseconds) | |
277 | ||
278 | if seconds > 60: | |
279 | self._timestamps.popleft() | |
280 | count -= 1 | |
281 | ||
282 | def ClearLine(self, length): | |
283 | """Clear any characters on the current line | |
284 | ||
285 | Make way for a new line of length 'length', by outputting enough | |
286 | spaces to clear out the old line. Then remember the new length for | |
287 | next time. | |
288 | ||
289 | Args: | |
290 | length: Length of new line, in characters | |
291 | """ | |
292 | if length < self.last_line_len: | |
293 | print ' ' * (self.last_line_len - length), | |
294 | print '\r', | |
295 | self.last_line_len = length | |
296 | sys.stdout.flush() | |
297 | ||
298 | def SelectCommit(self, commit, checkout=True): | |
299 | """Checkout the selected commit for this build | |
300 | """ | |
301 | self.commit = commit | |
302 | if checkout and self.checkout: | |
303 | gitutil.Checkout(commit.hash) | |
304 | ||
305 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): | |
306 | """Run make | |
307 | ||
308 | Args: | |
309 | commit: Commit object that is being built | |
310 | brd: Board object that is being built | |
fd18a89e | 311 | stage: Stage that we are at (mrproper, config, build) |
fc3fe1c2 SG |
312 | cwd: Directory where make should be run |
313 | args: Arguments to pass to make | |
314 | kwargs: Arguments to pass to command.RunPipe() | |
315 | """ | |
99796923 | 316 | cmd = [self.gnu_make] + list(args) |
fc3fe1c2 SG |
317 | result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
318 | cwd=cwd, raise_on_error=False, **kwargs) | |
319 | return result | |
320 | ||
321 | def ProcessResult(self, result): | |
322 | """Process the result of a build, showing progress information | |
323 | ||
324 | Args: | |
e5a0e5d8 SG |
325 | result: A CommandResult object, which indicates the result for |
326 | a single build | |
fc3fe1c2 SG |
327 | """ |
328 | col = terminal.Color() | |
329 | if result: | |
330 | target = result.brd.target | |
331 | ||
332 | if result.return_code < 0: | |
333 | self.active = False | |
334 | command.StopAll() | |
335 | return | |
336 | ||
337 | self.upto += 1 | |
338 | if result.return_code != 0: | |
339 | self.fail += 1 | |
340 | elif result.stderr: | |
341 | self.warned += 1 | |
342 | if result.already_done: | |
343 | self.already_done += 1 | |
e5a0e5d8 SG |
344 | if self._verbose: |
345 | print '\r', | |
346 | self.ClearLine(0) | |
347 | boards_selected = {target : result.brd} | |
348 | self.ResetResultSummary(boards_selected) | |
349 | self.ProduceResultSummary(result.commit_upto, self.commits, | |
350 | boards_selected) | |
fc3fe1c2 SG |
351 | else: |
352 | target = '(starting)' | |
353 | ||
354 | # Display separate counts for ok, warned and fail | |
355 | ok = self.upto - self.warned - self.fail | |
356 | line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) | |
357 | line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) | |
358 | line += self.col.Color(self.col.RED, '%5d' % self.fail) | |
359 | ||
360 | name = ' /%-5d ' % self.count | |
361 | ||
362 | # Add our current completion time estimate | |
363 | self._AddTimestamp() | |
364 | if self._complete_delay: | |
365 | name += '%s : ' % self._complete_delay | |
366 | # When building all boards for a commit, we can print a commit | |
367 | # progress message. | |
368 | if result and result.commit_upto is None: | |
369 | name += 'commit %2d/%-3d' % (self.commit_upto + 1, | |
370 | self.commit_count) | |
371 | ||
372 | name += target | |
373 | print line + name, | |
e5a0e5d8 | 374 | length = 14 + len(name) |
fc3fe1c2 SG |
375 | self.ClearLine(length) |
376 | ||
377 | def _GetOutputDir(self, commit_upto): | |
378 | """Get the name of the output directory for a commit number | |
379 | ||
380 | The output directory is typically .../<branch>/<commit>. | |
381 | ||
382 | Args: | |
383 | commit_upto: Commit number to use (0..self.count-1) | |
384 | """ | |
fea5858e SG |
385 | if self.commits: |
386 | commit = self.commits[commit_upto] | |
387 | subject = commit.subject.translate(trans_valid_chars) | |
388 | commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, | |
389 | self.commit_count, commit.hash, subject[:20])) | |
390 | else: | |
391 | commit_dir = 'current' | |
fc3fe1c2 SG |
392 | output_dir = os.path.join(self.base_dir, commit_dir) |
393 | return output_dir | |
394 | ||
395 | def GetBuildDir(self, commit_upto, target): | |
396 | """Get the name of the build directory for a commit number | |
397 | ||
398 | The build directory is typically .../<branch>/<commit>/<target>. | |
399 | ||
400 | Args: | |
401 | commit_upto: Commit number to use (0..self.count-1) | |
402 | target: Target name | |
403 | """ | |
404 | output_dir = self._GetOutputDir(commit_upto) | |
405 | return os.path.join(output_dir, target) | |
406 | ||
407 | def GetDoneFile(self, commit_upto, target): | |
408 | """Get the name of the done file for a commit number | |
409 | ||
410 | Args: | |
411 | commit_upto: Commit number to use (0..self.count-1) | |
412 | target: Target name | |
413 | """ | |
414 | return os.path.join(self.GetBuildDir(commit_upto, target), 'done') | |
415 | ||
416 | def GetSizesFile(self, commit_upto, target): | |
417 | """Get the name of the sizes file for a commit number | |
418 | ||
419 | Args: | |
420 | commit_upto: Commit number to use (0..self.count-1) | |
421 | target: Target name | |
422 | """ | |
423 | return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') | |
424 | ||
425 | def GetFuncSizesFile(self, commit_upto, target, elf_fname): | |
426 | """Get the name of the funcsizes file for a commit number and ELF file | |
427 | ||
428 | Args: | |
429 | commit_upto: Commit number to use (0..self.count-1) | |
430 | target: Target name | |
431 | elf_fname: Filename of elf image | |
432 | """ | |
433 | return os.path.join(self.GetBuildDir(commit_upto, target), | |
434 | '%s.sizes' % elf_fname.replace('/', '-')) | |
435 | ||
436 | def GetObjdumpFile(self, commit_upto, target, elf_fname): | |
437 | """Get the name of the objdump file for a commit number and ELF file | |
438 | ||
439 | Args: | |
440 | commit_upto: Commit number to use (0..self.count-1) | |
441 | target: Target name | |
442 | elf_fname: Filename of elf image | |
443 | """ | |
444 | return os.path.join(self.GetBuildDir(commit_upto, target), | |
445 | '%s.objdump' % elf_fname.replace('/', '-')) | |
446 | ||
447 | def GetErrFile(self, commit_upto, target): | |
448 | """Get the name of the err file for a commit number | |
449 | ||
450 | Args: | |
451 | commit_upto: Commit number to use (0..self.count-1) | |
452 | target: Target name | |
453 | """ | |
454 | output_dir = self.GetBuildDir(commit_upto, target) | |
455 | return os.path.join(output_dir, 'err') | |
456 | ||
457 | def FilterErrors(self, lines): | |
458 | """Filter out errors in which we have no interest | |
459 | ||
460 | We should probably use map(). | |
461 | ||
462 | Args: | |
463 | lines: List of error lines, each a string | |
464 | Returns: | |
465 | New list with only interesting lines included | |
466 | """ | |
467 | out_lines = [] | |
468 | for line in lines: | |
469 | if not self.re_make_err.search(line): | |
470 | out_lines.append(line) | |
471 | return out_lines | |
472 | ||
473 | def ReadFuncSizes(self, fname, fd): | |
474 | """Read function sizes from the output of 'nm' | |
475 | ||
476 | Args: | |
477 | fd: File containing data to read | |
478 | fname: Filename we are reading from (just for errors) | |
479 | ||
480 | Returns: | |
481 | Dictionary containing size of each function in bytes, indexed by | |
482 | function name. | |
483 | """ | |
484 | sym = {} | |
485 | for line in fd.readlines(): | |
486 | try: | |
487 | size, type, name = line[:-1].split() | |
488 | except: | |
489 | print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) | |
490 | continue | |
491 | if type in 'tTdDbB': | |
492 | # function names begin with '.' on 64-bit powerpc | |
493 | if '.' in name[1:]: | |
494 | name = 'static.' + name.split('.')[0] | |
495 | sym[name] = sym.get(name, 0) + int(size, 16) | |
496 | return sym | |
497 | ||
498 | def GetBuildOutcome(self, commit_upto, target, read_func_sizes): | |
499 | """Work out the outcome of a build. | |
500 | ||
501 | Args: | |
502 | commit_upto: Commit number to check (0..n-1) | |
503 | target: Target board to check | |
504 | read_func_sizes: True to read function size information | |
505 | ||
506 | Returns: | |
507 | Outcome object | |
508 | """ | |
509 | done_file = self.GetDoneFile(commit_upto, target) | |
510 | sizes_file = self.GetSizesFile(commit_upto, target) | |
511 | sizes = {} | |
512 | func_sizes = {} | |
513 | if os.path.exists(done_file): | |
514 | with open(done_file, 'r') as fd: | |
515 | return_code = int(fd.readline()) | |
516 | err_lines = [] | |
517 | err_file = self.GetErrFile(commit_upto, target) | |
518 | if os.path.exists(err_file): | |
519 | with open(err_file, 'r') as fd: | |
520 | err_lines = self.FilterErrors(fd.readlines()) | |
521 | ||
522 | # Decide whether the build was ok, failed or created warnings | |
523 | if return_code: | |
524 | rc = OUTCOME_ERROR | |
525 | elif len(err_lines): | |
526 | rc = OUTCOME_WARNING | |
527 | else: | |
528 | rc = OUTCOME_OK | |
529 | ||
530 | # Convert size information to our simple format | |
531 | if os.path.exists(sizes_file): | |
532 | with open(sizes_file, 'r') as fd: | |
533 | for line in fd.readlines(): | |
534 | values = line.split() | |
535 | rodata = 0 | |
536 | if len(values) > 6: | |
537 | rodata = int(values[6], 16) | |
538 | size_dict = { | |
539 | 'all' : int(values[0]) + int(values[1]) + | |
540 | int(values[2]), | |
541 | 'text' : int(values[0]) - rodata, | |
542 | 'data' : int(values[1]), | |
543 | 'bss' : int(values[2]), | |
544 | 'rodata' : rodata, | |
545 | } | |
546 | sizes[values[5]] = size_dict | |
547 | ||
548 | if read_func_sizes: | |
549 | pattern = self.GetFuncSizesFile(commit_upto, target, '*') | |
550 | for fname in glob.glob(pattern): | |
551 | with open(fname, 'r') as fd: | |
552 | dict_name = os.path.basename(fname).replace('.sizes', | |
553 | '') | |
554 | func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) | |
555 | ||
556 | return Builder.Outcome(rc, err_lines, sizes, func_sizes) | |
557 | ||
558 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) | |
559 | ||
560 | def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): | |
561 | """Calculate a summary of the results of building a commit. | |
562 | ||
563 | Args: | |
564 | board_selected: Dict containing boards to summarise | |
565 | commit_upto: Commit number to summarize (0..self.count-1) | |
566 | read_func_sizes: True to read function size information | |
567 | ||
568 | Returns: | |
569 | Tuple: | |
570 | Dict containing boards which passed building this commit. | |
571 | keyed by board.target | |
572 | List containing a summary of error/warning lines | |
573 | """ | |
574 | board_dict = {} | |
575 | err_lines_summary = [] | |
576 | ||
577 | for board in boards_selected.itervalues(): | |
578 | outcome = self.GetBuildOutcome(commit_upto, board.target, | |
579 | read_func_sizes) | |
580 | board_dict[board.target] = outcome | |
581 | for err in outcome.err_lines: | |
582 | if err and not err.rstrip() in err_lines_summary: | |
583 | err_lines_summary.append(err.rstrip()) | |
584 | return board_dict, err_lines_summary | |
585 | ||
586 | def AddOutcome(self, board_dict, arch_list, changes, char, color): | |
587 | """Add an output to our list of outcomes for each architecture | |
588 | ||
589 | This simple function adds failing boards (changes) to the | |
590 | relevant architecture string, so we can print the results out | |
591 | sorted by architecture. | |
592 | ||
593 | Args: | |
594 | board_dict: Dict containing all boards | |
595 | arch_list: Dict keyed by arch name. Value is a string containing | |
596 | a list of board names which failed for that arch. | |
597 | changes: List of boards to add to arch_list | |
598 | color: terminal.Colour object | |
599 | """ | |
600 | done_arch = {} | |
601 | for target in changes: | |
602 | if target in board_dict: | |
603 | arch = board_dict[target].arch | |
604 | else: | |
605 | arch = 'unknown' | |
606 | str = self.col.Color(color, ' ' + target) | |
607 | if not arch in done_arch: | |
608 | str = self.col.Color(color, char) + ' ' + str | |
609 | done_arch[arch] = True | |
610 | if not arch in arch_list: | |
611 | arch_list[arch] = str | |
612 | else: | |
613 | arch_list[arch] += str | |
614 | ||
615 | ||
616 | def ColourNum(self, num): | |
617 | color = self.col.RED if num > 0 else self.col.GREEN | |
618 | if num == 0: | |
619 | return '0' | |
620 | return self.col.Color(color, str(num)) | |
621 | ||
622 | def ResetResultSummary(self, board_selected): | |
623 | """Reset the results summary ready for use. | |
624 | ||
625 | Set up the base board list to be all those selected, and set the | |
626 | error lines to empty. | |
627 | ||
628 | Following this, calls to PrintResultSummary() will use this | |
629 | information to work out what has changed. | |
630 | ||
631 | Args: | |
632 | board_selected: Dict containing boards to summarise, keyed by | |
633 | board.target | |
634 | """ | |
635 | self._base_board_dict = {} | |
636 | for board in board_selected: | |
637 | self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) | |
638 | self._base_err_lines = [] | |
639 | ||
640 | def PrintFuncSizeDetail(self, fname, old, new): | |
641 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 | |
642 | delta, common = [], {} | |
643 | ||
644 | for a in old: | |
645 | if a in new: | |
646 | common[a] = 1 | |
647 | ||
648 | for name in old: | |
649 | if name not in common: | |
650 | remove += 1 | |
651 | down += old[name] | |
652 | delta.append([-old[name], name]) | |
653 | ||
654 | for name in new: | |
655 | if name not in common: | |
656 | add += 1 | |
657 | up += new[name] | |
658 | delta.append([new[name], name]) | |
659 | ||
660 | for name in common: | |
661 | diff = new.get(name, 0) - old.get(name, 0) | |
662 | if diff > 0: | |
663 | grow, up = grow + 1, up + diff | |
664 | elif diff < 0: | |
665 | shrink, down = shrink + 1, down - diff | |
666 | delta.append([diff, name]) | |
667 | ||
668 | delta.sort() | |
669 | delta.reverse() | |
670 | ||
671 | args = [add, -remove, grow, -shrink, up, -down, up - down] | |
672 | if max(args) == 0: | |
673 | return | |
674 | args = [self.ColourNum(x) for x in args] | |
675 | indent = ' ' * 15 | |
676 | print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % | |
677 | tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) | |
678 | print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', | |
679 | 'delta') | |
680 | for diff, name in delta: | |
681 | if diff: | |
682 | color = self.col.RED if diff > 0 else self.col.GREEN | |
683 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, | |
684 | old.get(name, '-'), new.get(name,'-'), diff) | |
685 | print self.col.Color(color, msg) | |
686 | ||
687 | ||
688 | def PrintSizeDetail(self, target_list, show_bloat): | |
689 | """Show details size information for each board | |
690 | ||
691 | Args: | |
692 | target_list: List of targets, each a dict containing: | |
693 | 'target': Target name | |
694 | 'total_diff': Total difference in bytes across all areas | |
695 | <part_name>: Difference for that part | |
696 | show_bloat: Show detail for each function | |
697 | """ | |
698 | targets_by_diff = sorted(target_list, reverse=True, | |
699 | key=lambda x: x['_total_diff']) | |
700 | for result in targets_by_diff: | |
701 | printed_target = False | |
702 | for name in sorted(result): | |
703 | diff = result[name] | |
704 | if name.startswith('_'): | |
705 | continue | |
706 | if diff != 0: | |
707 | color = self.col.RED if diff > 0 else self.col.GREEN | |
708 | msg = ' %s %+d' % (name, diff) | |
709 | if not printed_target: | |
710 | print '%10s %-15s:' % ('', result['_target']), | |
711 | printed_target = True | |
712 | print self.col.Color(color, msg), | |
713 | if printed_target: | |
714 | ||
715 | if show_bloat: | |
716 | target = result['_target'] | |
717 | outcome = result['_outcome'] | |
718 | base_outcome = self._base_board_dict[target] | |
719 | for fname in outcome.func_sizes: | |
720 | self.PrintFuncSizeDetail(fname, | |
721 | base_outcome.func_sizes[fname], | |
722 | outcome.func_sizes[fname]) | |
723 | ||
724 | ||
725 | def PrintSizeSummary(self, board_selected, board_dict, show_detail, | |
726 | show_bloat): | |
727 | """Print a summary of image sizes broken down by section. | |
728 | ||
729 | The summary takes the form of one line per architecture. The | |
730 | line contains deltas for each of the sections (+ means the section | |
731 | got bigger, - means smaller). The nunmbers are the average number | |
732 | of bytes that a board in this section increased by. | |
733 | ||
734 | For example: | |
735 | powerpc: (622 boards) text -0.0 | |
736 | arm: (285 boards) text -0.0 | |
737 | nds32: (3 boards) text -8.0 | |
738 | ||
739 | Args: | |
740 | board_selected: Dict containing boards to summarise, keyed by | |
741 | board.target | |
742 | board_dict: Dict containing boards for which we built this | |
743 | commit, keyed by board.target. The value is an Outcome object. | |
744 | show_detail: Show detail for each board | |
745 | show_bloat: Show detail for each function | |
746 | """ | |
747 | arch_list = {} | |
748 | arch_count = {} | |
749 | ||
750 | # Calculate changes in size for different image parts | |
751 | # The previous sizes are in Board.sizes, for each board | |
752 | for target in board_dict: | |
753 | if target not in board_selected: | |
754 | continue | |
755 | base_sizes = self._base_board_dict[target].sizes | |
756 | outcome = board_dict[target] | |
757 | sizes = outcome.sizes | |
758 | ||
759 | # Loop through the list of images, creating a dict of size | |
760 | # changes for each image/part. We end up with something like | |
761 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} | |
762 | # which means that U-Boot data increased by 5 bytes and SPL | |
763 | # text decreased by 4. | |
764 | err = {'_target' : target} | |
765 | for image in sizes: | |
766 | if image in base_sizes: | |
767 | base_image = base_sizes[image] | |
768 | # Loop through the text, data, bss parts | |
769 | for part in sorted(sizes[image]): | |
770 | diff = sizes[image][part] - base_image[part] | |
771 | col = None | |
772 | if diff: | |
773 | if image == 'u-boot': | |
774 | name = part | |
775 | else: | |
776 | name = image + ':' + part | |
777 | err[name] = diff | |
778 | arch = board_selected[target].arch | |
779 | if not arch in arch_count: | |
780 | arch_count[arch] = 1 | |
781 | else: | |
782 | arch_count[arch] += 1 | |
783 | if not sizes: | |
784 | pass # Only add to our list when we have some stats | |
785 | elif not arch in arch_list: | |
786 | arch_list[arch] = [err] | |
787 | else: | |
788 | arch_list[arch].append(err) | |
789 | ||
790 | # We now have a list of image size changes sorted by arch | |
791 | # Print out a summary of these | |
792 | for arch, target_list in arch_list.iteritems(): | |
793 | # Get total difference for each type | |
794 | totals = {} | |
795 | for result in target_list: | |
796 | total = 0 | |
797 | for name, diff in result.iteritems(): | |
798 | if name.startswith('_'): | |
799 | continue | |
800 | total += diff | |
801 | if name in totals: | |
802 | totals[name] += diff | |
803 | else: | |
804 | totals[name] = diff | |
805 | result['_total_diff'] = total | |
806 | result['_outcome'] = board_dict[result['_target']] | |
807 | ||
808 | count = len(target_list) | |
809 | printed_arch = False | |
810 | for name in sorted(totals): | |
811 | diff = totals[name] | |
812 | if diff: | |
813 | # Display the average difference in this name for this | |
814 | # architecture | |
815 | avg_diff = float(diff) / count | |
816 | color = self.col.RED if avg_diff > 0 else self.col.GREEN | |
817 | msg = ' %s %+1.1f' % (name, avg_diff) | |
818 | if not printed_arch: | |
819 | print '%10s: (for %d/%d boards)' % (arch, count, | |
820 | arch_count[arch]), | |
821 | printed_arch = True | |
822 | print self.col.Color(color, msg), | |
823 | ||
824 | if printed_arch: | |
825 | ||
826 | if show_detail: | |
827 | self.PrintSizeDetail(target_list, show_bloat) | |
828 | ||
829 | ||
830 | def PrintResultSummary(self, board_selected, board_dict, err_lines, | |
831 | show_sizes, show_detail, show_bloat): | |
832 | """Compare results with the base results and display delta. | |
833 | ||
834 | Only boards mentioned in board_selected will be considered. This | |
835 | function is intended to be called repeatedly with the results of | |
836 | each commit. It therefore shows a 'diff' between what it saw in | |
837 | the last call and what it sees now. | |
838 | ||
839 | Args: | |
840 | board_selected: Dict containing boards to summarise, keyed by | |
841 | board.target | |
842 | board_dict: Dict containing boards for which we built this | |
843 | commit, keyed by board.target. The value is an Outcome object. | |
844 | err_lines: A list of errors for this commit, or [] if there is | |
845 | none, or we don't want to print errors | |
846 | show_sizes: Show image size deltas | |
847 | show_detail: Show detail for each board | |
848 | show_bloat: Show detail for each function | |
849 | """ | |
850 | better = [] # List of boards fixed since last commit | |
851 | worse = [] # List of new broken boards since last commit | |
852 | new = [] # List of boards that didn't exist last time | |
853 | unknown = [] # List of boards that were not built | |
854 | ||
855 | for target in board_dict: | |
856 | if target not in board_selected: | |
857 | continue | |
858 | ||
859 | # If the board was built last time, add its outcome to a list | |
860 | if target in self._base_board_dict: | |
861 | base_outcome = self._base_board_dict[target].rc | |
862 | outcome = board_dict[target] | |
863 | if outcome.rc == OUTCOME_UNKNOWN: | |
864 | unknown.append(target) | |
865 | elif outcome.rc < base_outcome: | |
866 | better.append(target) | |
867 | elif outcome.rc > base_outcome: | |
868 | worse.append(target) | |
869 | else: | |
870 | new.append(target) | |
871 | ||
872 | # Get a list of errors that have appeared, and disappeared | |
873 | better_err = [] | |
874 | worse_err = [] | |
875 | for line in err_lines: | |
876 | if line not in self._base_err_lines: | |
877 | worse_err.append('+' + line) | |
878 | for line in self._base_err_lines: | |
879 | if line not in err_lines: | |
880 | better_err.append('-' + line) | |
881 | ||
882 | # Display results by arch | |
883 | if better or worse or unknown or new or worse_err or better_err: | |
884 | arch_list = {} | |
885 | self.AddOutcome(board_selected, arch_list, better, '', | |
886 | self.col.GREEN) | |
887 | self.AddOutcome(board_selected, arch_list, worse, '+', | |
888 | self.col.RED) | |
889 | self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) | |
890 | if self._show_unknown: | |
891 | self.AddOutcome(board_selected, arch_list, unknown, '?', | |
892 | self.col.MAGENTA) | |
893 | for arch, target_list in arch_list.iteritems(): | |
894 | print '%10s: %s' % (arch, target_list) | |
28370c1b | 895 | self._error_lines += 1 |
fc3fe1c2 SG |
896 | if better_err: |
897 | print self.col.Color(self.col.GREEN, '\n'.join(better_err)) | |
28370c1b | 898 | self._error_lines += 1 |
fc3fe1c2 SG |
899 | if worse_err: |
900 | print self.col.Color(self.col.RED, '\n'.join(worse_err)) | |
28370c1b | 901 | self._error_lines += 1 |
fc3fe1c2 SG |
902 | |
903 | if show_sizes: | |
904 | self.PrintSizeSummary(board_selected, board_dict, show_detail, | |
905 | show_bloat) | |
906 | ||
907 | # Save our updated information for the next call to this function | |
908 | self._base_board_dict = board_dict | |
909 | self._base_err_lines = err_lines | |
910 | ||
911 | # Get a list of boards that did not get built, if needed | |
912 | not_built = [] | |
913 | for board in board_selected: | |
914 | if not board in board_dict: | |
915 | not_built.append(board) | |
916 | if not_built: | |
917 | print "Boards not built (%d): %s" % (len(not_built), | |
918 | ', '.join(not_built)) | |
919 | ||
b2ea7ab2 SG |
920 | def ProduceResultSummary(self, commit_upto, commits, board_selected): |
921 | board_dict, err_lines = self.GetResultSummary(board_selected, | |
922 | commit_upto, read_func_sizes=self._show_bloat) | |
923 | if commits: | |
924 | msg = '%02d: %s' % (commit_upto + 1, | |
925 | commits[commit_upto].subject) | |
926 | print self.col.Color(self.col.BLUE, msg) | |
927 | self.PrintResultSummary(board_selected, board_dict, | |
928 | err_lines if self._show_errors else [], | |
929 | self._show_sizes, self._show_detail, self._show_bloat) | |
fc3fe1c2 | 930 | |
b2ea7ab2 | 931 | def ShowSummary(self, commits, board_selected): |
fc3fe1c2 SG |
932 | """Show a build summary for U-Boot for a given board list. |
933 | ||
934 | Reset the result summary, then repeatedly call GetResultSummary on | |
935 | each commit's results, then display the differences we see. | |
936 | ||
937 | Args: | |
938 | commit: Commit objects to summarise | |
939 | board_selected: Dict containing boards to summarise | |
fc3fe1c2 | 940 | """ |
fea5858e | 941 | self.commit_count = len(commits) if commits else 1 |
fc3fe1c2 SG |
942 | self.commits = commits |
943 | self.ResetResultSummary(board_selected) | |
28370c1b | 944 | self._error_lines = 0 |
fc3fe1c2 SG |
945 | |
946 | for commit_upto in range(0, self.commit_count, self._step): | |
b2ea7ab2 | 947 | self.ProduceResultSummary(commit_upto, commits, board_selected) |
28370c1b SG |
948 | if not self._error_lines: |
949 | print self.col.Color(self.col.GREEN, '(no errors to report)') | |
fc3fe1c2 SG |
950 | |
951 | ||
952 | def SetupBuild(self, board_selected, commits): | |
953 | """Set up ready to start a build. | |
954 | ||
955 | Args: | |
956 | board_selected: Selected boards to build | |
957 | commits: Selected commits to build | |
958 | """ | |
959 | # First work out how many commits we will build | |
fea5858e | 960 | count = (self.commit_count + self._step - 1) / self._step |
fc3fe1c2 SG |
961 | self.count = len(board_selected) * count |
962 | self.upto = self.warned = self.fail = 0 | |
963 | self._timestamps = collections.deque() | |
964 | ||
fc3fe1c2 SG |
965 | def GetThreadDir(self, thread_num): |
966 | """Get the directory path to the working dir for a thread. | |
967 | ||
968 | Args: | |
969 | thread_num: Number of thread to check. | |
970 | """ | |
971 | return os.path.join(self._working_dir, '%02d' % thread_num) | |
972 | ||
fea5858e | 973 | def _PrepareThread(self, thread_num, setup_git): |
fc3fe1c2 SG |
974 | """Prepare the working directory for a thread. |
975 | ||
976 | This clones or fetches the repo into the thread's work directory. | |
977 | ||
978 | Args: | |
979 | thread_num: Thread number (0, 1, ...) | |
fea5858e | 980 | setup_git: True to set up a git repo clone |
fc3fe1c2 SG |
981 | """ |
982 | thread_dir = self.GetThreadDir(thread_num) | |
190064b4 | 983 | builderthread.Mkdir(thread_dir) |
fc3fe1c2 SG |
984 | git_dir = os.path.join(thread_dir, '.git') |
985 | ||
986 | # Clone the repo if it doesn't already exist | |
987 | # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so | |
988 | # we have a private index but uses the origin repo's contents? | |
fea5858e | 989 | if setup_git and self.git_dir: |
fc3fe1c2 SG |
990 | src_dir = os.path.abspath(self.git_dir) |
991 | if os.path.exists(git_dir): | |
992 | gitutil.Fetch(git_dir, thread_dir) | |
993 | else: | |
994 | print 'Cloning repo for thread %d' % thread_num | |
995 | gitutil.Clone(src_dir, thread_dir) | |
996 | ||
fea5858e | 997 | def _PrepareWorkingSpace(self, max_threads, setup_git): |
fc3fe1c2 SG |
998 | """Prepare the working directory for use. |
999 | ||
1000 | Set up the git repo for each thread. | |
1001 | ||
1002 | Args: | |
1003 | max_threads: Maximum number of threads we expect to need. | |
fea5858e | 1004 | setup_git: True to set up a git repo clone |
fc3fe1c2 | 1005 | """ |
190064b4 | 1006 | builderthread.Mkdir(self._working_dir) |
fc3fe1c2 | 1007 | for thread in range(max_threads): |
fea5858e | 1008 | self._PrepareThread(thread, setup_git) |
fc3fe1c2 SG |
1009 | |
1010 | def _PrepareOutputSpace(self): | |
1011 | """Get the output directories ready to receive files. | |
1012 | ||
1013 | We delete any output directories which look like ones we need to | |
1014 | create. Having left over directories is confusing when the user wants | |
1015 | to check the output manually. | |
1016 | """ | |
1017 | dir_list = [] | |
1018 | for commit_upto in range(self.commit_count): | |
1019 | dir_list.append(self._GetOutputDir(commit_upto)) | |
1020 | ||
1021 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): | |
1022 | if dirname not in dir_list: | |
1023 | shutil.rmtree(dirname) | |
1024 | ||
e5a0e5d8 | 1025 | def BuildBoards(self, commits, board_selected, keep_outputs, verbose): |
fc3fe1c2 SG |
1026 | """Build all commits for a list of boards |
1027 | ||
1028 | Args: | |
1029 | commits: List of commits to be build, each a Commit object | |
1030 | boards_selected: Dict of selected boards, key is target name, | |
1031 | value is Board object | |
fc3fe1c2 | 1032 | keep_outputs: True to save build output files |
e5a0e5d8 | 1033 | verbose: Display build results as they are completed |
fc3fe1c2 | 1034 | """ |
fea5858e | 1035 | self.commit_count = len(commits) if commits else 1 |
fc3fe1c2 | 1036 | self.commits = commits |
e5a0e5d8 | 1037 | self._verbose = verbose |
fc3fe1c2 SG |
1038 | |
1039 | self.ResetResultSummary(board_selected) | |
190064b4 | 1040 | builderthread.Mkdir(self.base_dir) |
fea5858e SG |
1041 | self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), |
1042 | commits is not None) | |
fc3fe1c2 SG |
1043 | self._PrepareOutputSpace() |
1044 | self.SetupBuild(board_selected, commits) | |
1045 | self.ProcessResult(None) | |
1046 | ||
1047 | # Create jobs to build all commits for each board | |
1048 | for brd in board_selected.itervalues(): | |
190064b4 | 1049 | job = builderthread.BuilderJob() |
fc3fe1c2 SG |
1050 | job.board = brd |
1051 | job.commits = commits | |
1052 | job.keep_outputs = keep_outputs | |
1053 | job.step = self._step | |
1054 | self.queue.put(job) | |
1055 | ||
1056 | # Wait until all jobs are started | |
1057 | self.queue.join() | |
1058 | ||
1059 | # Wait until we have processed all output | |
1060 | self.out_queue.join() | |
1061 | ||
1062 | self.ClearLine(0) |