]>
git.ipfire.org Git - people/ms/u-boot.git/blob - tools/buildman/builder.py
1 # Copyright (c) 2013 The Chromium OS Authors.
3 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 from datetime
import datetime
, timedelta
23 from terminal
import Print
30 Please see README for user documentation, and you should be familiar with
31 that before trying to make sense of this.
33 Buildman works by keeping the machine as busy as possible, building different
34 commits for different boards on multiple CPUs at once.
36 The source repo (self.git_dir) contains all the commits to be built. Each
37 thread works on a single board at a time. It checks out the first commit,
38 configures it for that board, then builds it. Then it checks out the next
39 commit and builds it (typically without re-configuring). When it runs out
40 of commits, it gets another job from the builder and starts again with that
43 Clearly the builder threads could work either way - they could check out a
44 commit and then built it for all boards. Using separate directories for each
45 commit/board pair they could leave their build product around afterwards
48 The intent behind building a single board for multiple commits, is to make
49 use of incremental builds. Since each commit is built incrementally from
50 the previous one, builds are faster. Reconfiguring for a different board
51 removes all intermediate object files.
53 Many threads can be working at once, but each has its own working directory.
54 When a thread finishes a build, it puts the output files into a result
57 The base directory used by buildman is normally '../<branch>', i.e.
58 a directory higher than the source repository and named after the branch
61 Within the base directory, we have one subdirectory for each commit. Within
62 that is one subdirectory for each board. Within that is the build output for
63 that commit/board combination.
65 Buildman also create working directories for each thread, in a .bm-work/
66 subdirectory in the base dir.
68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
69 'seaboard', and say that us-net has two commits. We will have directories
72 us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
84 00/ working directory for thread 0 (contains source checkout)
86 01/ working directory for thread 1
89 u-boot/ source directory
93 # Possible build outcomes
94 OUTCOME_OK
, OUTCOME_WARNING
, OUTCOME_ERROR
, OUTCOME_UNKNOWN
= range(4)
96 # Translate a commit subject into a valid filename
97 trans_valid_chars
= string
.maketrans("/: ", "---")
101 """Class for building U-Boot for a particular commit.
103 Public members: (many should ->private)
104 active: True if the builder is active and has not been stopped
105 already_done: Number of builds already completed
106 base_dir: Base directory to use for builder
107 checkout: True to check out source, False to skip that step.
108 This is used for testing.
109 col: terminal.Color() object
110 count: Number of commits to build
111 do_make: Method to call to invoke Make
112 fail: Number of builds that failed due to error
113 force_build: Force building even if a build already exists
114 force_config_on_failure: If a commit fails for a board, disable
115 incremental building for the next commit we build for that
116 board, so that we will see all warnings/errors again.
117 force_build_failures: If a previously-built build (i.e. built on
118 a previous run of buildman) is marked as failed, rebuild it.
119 git_dir: Git directory containing source repository
120 last_line_len: Length of the last line we printed (used for erasing
121 it with new progress information)
122 num_jobs: Number of jobs to run at once (passed to make as -j)
123 num_threads: Number of builder threads to run
124 out_queue: Queue of results to process
125 re_make_err: Compiled regular expression for ignore_lines
126 queue: Queue of jobs to run
127 threads: List of active threads
128 toolchains: Toolchains object to use for building
129 upto: Current commit number we are building (0.count-1)
130 warned: Number of builds that produced at least one warning
131 force_reconfig: Reconfigure U-Boot on each comiit. This disables
132 incremental building, where buildman reconfigures on the first
133 commit for a baord, and then just does an incremental build for
134 the following commits. In fact buildman will reconfigure and
135 retry for any failing commits, so generally the only effect of
136 this option is to slow things down.
137 in_tree: Build U-Boot in-tree instead of specifying an output
138 directory separate from the source code. This option is really
139 only useful for testing in-tree builds.
142 _base_board_dict: Last-summarised Dict of boards
143 _base_err_lines: Last-summarised list of errors
144 _base_warn_lines: Last-summarised list of warnings
145 _build_period_us: Time taken for a single build (float object).
146 _complete_delay: Expected delay until completion (timedelta)
147 _next_delay_update: Next time we plan to display a progress update
149 _show_unknown: Show unknown boards (those not built) in summary
150 _timestamps: List of timestamps for the completion of the last
151 last _timestamp_count builds. Each is a datetime object.
152 _timestamp_count: Number of timestamps to keep in our list.
153 _working_dir: Base working directory containing all threads
156 """Records a build outcome for a single make invocation
159 rc: Outcome value (OUTCOME_...)
160 err_lines: List of error lines or [] if none
161 sizes: Dictionary of image size information, keyed by filename
162 - Each value is itself a dictionary containing
163 values for 'text', 'data' and 'bss', being the integer
164 size in bytes of each section.
165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
166 value is itself a dictionary:
168 value: Size of function in bytes
170 def __init__(self
, rc
, err_lines
, sizes
, func_sizes
):
172 self
.err_lines
= err_lines
174 self
.func_sizes
= func_sizes
176 def __init__(self
, toolchains
, base_dir
, git_dir
, num_threads
, num_jobs
,
177 gnu_make
='make', checkout
=True, show_unknown
=True, step
=1,
178 no_subdirs
=False, full_path
=False):
179 """Create a new Builder object
182 toolchains: Toolchains object to use for building
183 base_dir: Base directory to use for builder
184 git_dir: Git directory containing source repository
185 num_threads: Number of builder threads to run
186 num_jobs: Number of jobs to run at once (passed to make as -j)
187 gnu_make: the command name of GNU Make.
188 checkout: True to check out source, False to skip that step.
189 This is used for testing.
190 show_unknown: Show unknown boards (those not built) in summary
191 step: 1 to process every commit, n to process every nth commit
192 no_subdirs: Don't create subdirectories when building current
193 source for a single board
194 full_path: Return the full path in CROSS_COMPILE and don't set
197 self
.toolchains
= toolchains
198 self
.base_dir
= base_dir
199 self
._working
_dir
= os
.path
.join(base_dir
, '.bm-work')
202 self
.do_make
= self
.Make
203 self
.gnu_make
= gnu_make
204 self
.checkout
= checkout
205 self
.num_threads
= num_threads
206 self
.num_jobs
= num_jobs
207 self
.already_done
= 0
208 self
.force_build
= False
209 self
.git_dir
= git_dir
210 self
._show
_unknown
= show_unknown
211 self
._timestamp
_count
= 10
212 self
._build
_period
_us
= None
213 self
._complete
_delay
= None
214 self
._next
_delay
_update
= datetime
.now()
215 self
.force_config_on_failure
= True
216 self
.force_build_failures
= False
217 self
.force_reconfig
= False
220 self
._error
_lines
= 0
221 self
.no_subdirs
= no_subdirs
222 self
.full_path
= full_path
224 self
.col
= terminal
.Color()
226 self
._re
_function
= re
.compile('(.*): In function.*')
227 self
._re
_files
= re
.compile('In file included from.*')
228 self
._re
_warning
= re
.compile('(.*):(\d*):(\d*): warning: .*')
229 self
._re
_note
= re
.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
231 self
.queue
= Queue
.Queue()
232 self
.out_queue
= Queue
.Queue()
233 for i
in range(self
.num_threads
):
234 t
= builderthread
.BuilderThread(self
, i
)
237 self
.threads
.append(t
)
239 self
.last_line_len
= 0
240 t
= builderthread
.ResultThread(self
)
243 self
.threads
.append(t
)
245 ignore_lines
= ['(make.*Waiting for unfinished)', '(Segmentation fault)']
246 self
.re_make_err
= re
.compile('|'.join(ignore_lines
))
249 """Get rid of all threads created by the builder"""
250 for t
in self
.threads
:
253 def SetDisplayOptions(self
, show_errors
=False, show_sizes
=False,
254 show_detail
=False, show_bloat
=False,
255 list_error_boards
=False):
256 """Setup display options for the builder.
258 show_errors: True to show summarised error/warning info
259 show_sizes: Show size deltas
260 show_detail: Show detail for each board
261 show_bloat: Show detail for each function
262 list_error_boards: Show the boards which caused each error/warning
264 self
._show
_errors
= show_errors
265 self
._show
_sizes
= show_sizes
266 self
._show
_detail
= show_detail
267 self
._show
_bloat
= show_bloat
268 self
._list
_error
_boards
= list_error_boards
270 def _AddTimestamp(self
):
271 """Add a new timestamp to the list and record the build period.
273 The build period is the length of time taken to perform a single
274 build (one board, one commit).
277 self
._timestamps
.append(now
)
278 count
= len(self
._timestamps
)
279 delta
= self
._timestamps
[-1] - self
._timestamps
[0]
280 seconds
= delta
.total_seconds()
282 # If we have enough data, estimate build period (time taken for a
283 # single build) and therefore completion time.
284 if count
> 1 and self
._next
_delay
_update
< now
:
285 self
._next
_delay
_update
= now
+ timedelta(seconds
=2)
287 self
._build
_period
= float(seconds
) / count
288 todo
= self
.count
- self
.upto
289 self
._complete
_delay
= timedelta(microseconds
=
290 self
._build
_period
* todo
* 1000000)
292 self
._complete
_delay
-= timedelta(
293 microseconds
=self
._complete
_delay
.microseconds
)
296 self
._timestamps
.popleft()
299 def ClearLine(self
, length
):
300 """Clear any characters on the current line
302 Make way for a new line of length 'length', by outputting enough
303 spaces to clear out the old line. Then remember the new length for
307 length: Length of new line, in characters
309 if length
< self
.last_line_len
:
310 Print(' ' * (self
.last_line_len
- length
), newline
=False)
311 Print('\r', newline
=False)
312 self
.last_line_len
= length
315 def SelectCommit(self
, commit
, checkout
=True):
316 """Checkout the selected commit for this build
319 if checkout
and self
.checkout
:
320 gitutil
.Checkout(commit
.hash)
322 def Make(self
, commit
, brd
, stage
, cwd
, *args
, **kwargs
):
326 commit: Commit object that is being built
327 brd: Board object that is being built
328 stage: Stage that we are at (mrproper, config, build)
329 cwd: Directory where make should be run
330 args: Arguments to pass to make
331 kwargs: Arguments to pass to command.RunPipe()
333 cmd
= [self
.gnu_make
] + list(args
)
334 result
= command
.RunPipe([cmd
], capture
=True, capture_stderr
=True,
335 cwd
=cwd
, raise_on_error
=False, **kwargs
)
338 def ProcessResult(self
, result
):
339 """Process the result of a build, showing progress information
342 result: A CommandResult object, which indicates the result for
345 col
= terminal
.Color()
347 target
= result
.brd
.target
349 if result
.return_code
< 0:
355 if result
.return_code
!= 0:
359 if result
.already_done
:
360 self
.already_done
+= 1
362 Print('\r', newline
=False)
364 boards_selected
= {target
: result
.brd
}
365 self
.ResetResultSummary(boards_selected
)
366 self
.ProduceResultSummary(result
.commit_upto
, self
.commits
,
369 target
= '(starting)'
371 # Display separate counts for ok, warned and fail
372 ok
= self
.upto
- self
.warned
- self
.fail
373 line
= '\r' + self
.col
.Color(self
.col
.GREEN
, '%5d' % ok
)
374 line
+= self
.col
.Color(self
.col
.YELLOW
, '%5d' % self
.warned
)
375 line
+= self
.col
.Color(self
.col
.RED
, '%5d' % self
.fail
)
377 name
= ' /%-5d ' % self
.count
379 # Add our current completion time estimate
381 if self
._complete
_delay
:
382 name
+= '%s : ' % self
._complete
_delay
383 # When building all boards for a commit, we can print a commit
385 if result
and result
.commit_upto
is None:
386 name
+= 'commit %2d/%-3d' % (self
.commit_upto
+ 1,
390 Print(line
+ name
, newline
=False)
391 length
= 14 + len(name
)
392 self
.ClearLine(length
)
394 def _GetOutputDir(self
, commit_upto
):
395 """Get the name of the output directory for a commit number
397 The output directory is typically .../<branch>/<commit>.
400 commit_upto: Commit number to use (0..self.count-1)
404 commit
= self
.commits
[commit_upto
]
405 subject
= commit
.subject
.translate(trans_valid_chars
)
406 commit_dir
= ('%02d_of_%02d_g%s_%s' % (commit_upto
+ 1,
407 self
.commit_count
, commit
.hash, subject
[:20]))
408 elif not self
.no_subdirs
:
409 commit_dir
= 'current'
412 return os
.path
.join(self
.base_dir
, commit_dir
)
414 def GetBuildDir(self
, commit_upto
, target
):
415 """Get the name of the build directory for a commit number
417 The build directory is typically .../<branch>/<commit>/<target>.
420 commit_upto: Commit number to use (0..self.count-1)
423 output_dir
= self
._GetOutputDir
(commit_upto
)
424 return os
.path
.join(output_dir
, target
)
426 def GetDoneFile(self
, commit_upto
, target
):
427 """Get the name of the done file for a commit number
430 commit_upto: Commit number to use (0..self.count-1)
433 return os
.path
.join(self
.GetBuildDir(commit_upto
, target
), 'done')
435 def GetSizesFile(self
, commit_upto
, target
):
436 """Get the name of the sizes file for a commit number
439 commit_upto: Commit number to use (0..self.count-1)
442 return os
.path
.join(self
.GetBuildDir(commit_upto
, target
), 'sizes')
444 def GetFuncSizesFile(self
, commit_upto
, target
, elf_fname
):
445 """Get the name of the funcsizes file for a commit number and ELF file
448 commit_upto: Commit number to use (0..self.count-1)
450 elf_fname: Filename of elf image
452 return os
.path
.join(self
.GetBuildDir(commit_upto
, target
),
453 '%s.sizes' % elf_fname
.replace('/', '-'))
455 def GetObjdumpFile(self
, commit_upto
, target
, elf_fname
):
456 """Get the name of the objdump file for a commit number and ELF file
459 commit_upto: Commit number to use (0..self.count-1)
461 elf_fname: Filename of elf image
463 return os
.path
.join(self
.GetBuildDir(commit_upto
, target
),
464 '%s.objdump' % elf_fname
.replace('/', '-'))
466 def GetErrFile(self
, commit_upto
, target
):
467 """Get the name of the err file for a commit number
470 commit_upto: Commit number to use (0..self.count-1)
473 output_dir
= self
.GetBuildDir(commit_upto
, target
)
474 return os
.path
.join(output_dir
, 'err')
476 def FilterErrors(self
, lines
):
477 """Filter out errors in which we have no interest
479 We should probably use map().
482 lines: List of error lines, each a string
484 New list with only interesting lines included
488 if not self
.re_make_err
.search(line
):
489 out_lines
.append(line
)
492 def ReadFuncSizes(self
, fname
, fd
):
493 """Read function sizes from the output of 'nm'
496 fd: File containing data to read
497 fname: Filename we are reading from (just for errors)
500 Dictionary containing size of each function in bytes, indexed by
504 for line
in fd
.readlines():
506 size
, type, name
= line
[:-1].split()
508 Print("Invalid line in file '%s': '%s'" % (fname
, line
[:-1]))
511 # function names begin with '.' on 64-bit powerpc
513 name
= 'static.' + name
.split('.')[0]
514 sym
[name
] = sym
.get(name
, 0) + int(size
, 16)
517 def GetBuildOutcome(self
, commit_upto
, target
, read_func_sizes
):
518 """Work out the outcome of a build.
521 commit_upto: Commit number to check (0..n-1)
522 target: Target board to check
523 read_func_sizes: True to read function size information
528 done_file
= self
.GetDoneFile(commit_upto
, target
)
529 sizes_file
= self
.GetSizesFile(commit_upto
, target
)
532 if os
.path
.exists(done_file
):
533 with
open(done_file
, 'r') as fd
:
534 return_code
= int(fd
.readline())
536 err_file
= self
.GetErrFile(commit_upto
, target
)
537 if os
.path
.exists(err_file
):
538 with
open(err_file
, 'r') as fd
:
539 err_lines
= self
.FilterErrors(fd
.readlines())
541 # Decide whether the build was ok, failed or created warnings
549 # Convert size information to our simple format
550 if os
.path
.exists(sizes_file
):
551 with
open(sizes_file
, 'r') as fd
:
552 for line
in fd
.readlines():
553 values
= line
.split()
556 rodata
= int(values
[6], 16)
558 'all' : int(values
[0]) + int(values
[1]) +
560 'text' : int(values
[0]) - rodata
,
561 'data' : int(values
[1]),
562 'bss' : int(values
[2]),
565 sizes
[values
[5]] = size_dict
568 pattern
= self
.GetFuncSizesFile(commit_upto
, target
, '*')
569 for fname
in glob
.glob(pattern
):
570 with
open(fname
, 'r') as fd
:
571 dict_name
= os
.path
.basename(fname
).replace('.sizes',
573 func_sizes
[dict_name
] = self
.ReadFuncSizes(fname
, fd
)
575 return Builder
.Outcome(rc
, err_lines
, sizes
, func_sizes
)
577 return Builder
.Outcome(OUTCOME_UNKNOWN
, [], {}, {})
579 def GetResultSummary(self
, boards_selected
, commit_upto
, read_func_sizes
):
580 """Calculate a summary of the results of building a commit.
583 board_selected: Dict containing boards to summarise
584 commit_upto: Commit number to summarize (0..self.count-1)
585 read_func_sizes: True to read function size information
589 Dict containing boards which passed building this commit.
590 keyed by board.target
591 List containing a summary of error lines
592 Dict keyed by error line, containing a list of the Board
593 objects with that error
594 List containing a summary of warning lines
595 Dict keyed by error line, containing a list of the Board
596 objects with that warning
598 def AddLine(lines_summary
, lines_boards
, line
, board
):
600 if line
in lines_boards
:
601 lines_boards
[line
].append(board
)
603 lines_boards
[line
] = [board
]
604 lines_summary
.append(line
)
607 err_lines_summary
= []
608 err_lines_boards
= {}
609 warn_lines_summary
= []
610 warn_lines_boards
= {}
612 for board
in boards_selected
.itervalues():
613 outcome
= self
.GetBuildOutcome(commit_upto
, board
.target
,
615 board_dict
[board
.target
] = outcome
617 last_was_warning
= False
618 for line
in outcome
.err_lines
:
620 if (self
._re
_function
.match(line
) or
621 self
._re
_files
.match(line
)):
624 is_warning
= self
._re
_warning
.match(line
)
625 is_note
= self
._re
_note
.match(line
)
626 if is_warning
or (last_was_warning
and is_note
):
628 AddLine(warn_lines_summary
, warn_lines_boards
,
630 AddLine(warn_lines_summary
, warn_lines_boards
,
634 AddLine(err_lines_summary
, err_lines_boards
,
636 AddLine(err_lines_summary
, err_lines_boards
,
638 last_was_warning
= is_warning
640 return (board_dict
, err_lines_summary
, err_lines_boards
,
641 warn_lines_summary
, warn_lines_boards
)
643 def AddOutcome(self
, board_dict
, arch_list
, changes
, char
, color
):
644 """Add an output to our list of outcomes for each architecture
646 This simple function adds failing boards (changes) to the
647 relevant architecture string, so we can print the results out
648 sorted by architecture.
651 board_dict: Dict containing all boards
652 arch_list: Dict keyed by arch name. Value is a string containing
653 a list of board names which failed for that arch.
654 changes: List of boards to add to arch_list
655 color: terminal.Colour object
658 for target
in changes
:
659 if target
in board_dict
:
660 arch
= board_dict
[target
].arch
663 str = self
.col
.Color(color
, ' ' + target
)
664 if not arch
in done_arch
:
665 str = self
.col
.Color(color
, char
) + ' ' + str
666 done_arch
[arch
] = True
667 if not arch
in arch_list
:
668 arch_list
[arch
] = str
670 arch_list
[arch
] += str
673 def ColourNum(self
, num
):
674 color
= self
.col
.RED
if num
> 0 else self
.col
.GREEN
677 return self
.col
.Color(color
, str(num
))
679 def ResetResultSummary(self
, board_selected
):
680 """Reset the results summary ready for use.
682 Set up the base board list to be all those selected, and set the
683 error lines to empty.
685 Following this, calls to PrintResultSummary() will use this
686 information to work out what has changed.
689 board_selected: Dict containing boards to summarise, keyed by
692 self
._base
_board
_dict
= {}
693 for board
in board_selected
:
694 self
._base
_board
_dict
[board
] = Builder
.Outcome(0, [], [], {})
695 self
._base
_err
_lines
= []
696 self
._base
_warn
_lines
= []
697 self
._base
_err
_line
_boards
= {}
698 self
._base
_warn
_line
_boards
= {}
700 def PrintFuncSizeDetail(self
, fname
, old
, new
):
701 grow
, shrink
, add
, remove
, up
, down
= 0, 0, 0, 0, 0, 0
702 delta
, common
= [], {}
709 if name
not in common
:
712 delta
.append([-old
[name
], name
])
715 if name
not in common
:
718 delta
.append([new
[name
], name
])
721 diff
= new
.get(name
, 0) - old
.get(name
, 0)
723 grow
, up
= grow
+ 1, up
+ diff
725 shrink
, down
= shrink
+ 1, down
- diff
726 delta
.append([diff
, name
])
731 args
= [add
, -remove
, grow
, -shrink
, up
, -down
, up
- down
]
734 args
= [self
.ColourNum(x
) for x
in args
]
736 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
737 tuple([indent
, self
.col
.Color(self
.col
.YELLOW
, fname
)] + args
))
738 Print('%s %-38s %7s %7s %+7s' % (indent
, 'function', 'old', 'new',
740 for diff
, name
in delta
:
742 color
= self
.col
.RED
if diff
> 0 else self
.col
.GREEN
743 msg
= '%s %-38s %7s %7s %+7d' % (indent
, name
,
744 old
.get(name
, '-'), new
.get(name
,'-'), diff
)
745 Print(msg
, colour
=color
)
748 def PrintSizeDetail(self
, target_list
, show_bloat
):
749 """Show details size information for each board
752 target_list: List of targets, each a dict containing:
753 'target': Target name
754 'total_diff': Total difference in bytes across all areas
755 <part_name>: Difference for that part
756 show_bloat: Show detail for each function
758 targets_by_diff
= sorted(target_list
, reverse
=True,
759 key
=lambda x
: x
['_total_diff'])
760 for result
in targets_by_diff
:
761 printed_target
= False
762 for name
in sorted(result
):
764 if name
.startswith('_'):
767 color
= self
.col
.RED
if diff
> 0 else self
.col
.GREEN
768 msg
= ' %s %+d' % (name
, diff
)
769 if not printed_target
:
770 Print('%10s %-15s:' % ('', result
['_target']),
772 printed_target
= True
773 Print(msg
, colour
=color
, newline
=False)
777 target
= result
['_target']
778 outcome
= result
['_outcome']
779 base_outcome
= self
._base
_board
_dict
[target
]
780 for fname
in outcome
.func_sizes
:
781 self
.PrintFuncSizeDetail(fname
,
782 base_outcome
.func_sizes
[fname
],
783 outcome
.func_sizes
[fname
])
786 def PrintSizeSummary(self
, board_selected
, board_dict
, show_detail
,
788 """Print a summary of image sizes broken down by section.
790 The summary takes the form of one line per architecture. The
791 line contains deltas for each of the sections (+ means the section
792 got bigger, - means smaller). The nunmbers are the average number
793 of bytes that a board in this section increased by.
796 powerpc: (622 boards) text -0.0
797 arm: (285 boards) text -0.0
798 nds32: (3 boards) text -8.0
801 board_selected: Dict containing boards to summarise, keyed by
803 board_dict: Dict containing boards for which we built this
804 commit, keyed by board.target. The value is an Outcome object.
805 show_detail: Show detail for each board
806 show_bloat: Show detail for each function
811 # Calculate changes in size for different image parts
812 # The previous sizes are in Board.sizes, for each board
813 for target
in board_dict
:
814 if target
not in board_selected
:
816 base_sizes
= self
._base
_board
_dict
[target
].sizes
817 outcome
= board_dict
[target
]
818 sizes
= outcome
.sizes
820 # Loop through the list of images, creating a dict of size
821 # changes for each image/part. We end up with something like
822 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
823 # which means that U-Boot data increased by 5 bytes and SPL
824 # text decreased by 4.
825 err
= {'_target' : target
}
827 if image
in base_sizes
:
828 base_image
= base_sizes
[image
]
829 # Loop through the text, data, bss parts
830 for part
in sorted(sizes
[image
]):
831 diff
= sizes
[image
][part
] - base_image
[part
]
834 if image
== 'u-boot':
837 name
= image
+ ':' + part
839 arch
= board_selected
[target
].arch
840 if not arch
in arch_count
:
843 arch_count
[arch
] += 1
845 pass # Only add to our list when we have some stats
846 elif not arch
in arch_list
:
847 arch_list
[arch
] = [err
]
849 arch_list
[arch
].append(err
)
851 # We now have a list of image size changes sorted by arch
852 # Print out a summary of these
853 for arch
, target_list
in arch_list
.iteritems():
854 # Get total difference for each type
856 for result
in target_list
:
858 for name
, diff
in result
.iteritems():
859 if name
.startswith('_'):
866 result
['_total_diff'] = total
867 result
['_outcome'] = board_dict
[result
['_target']]
869 count
= len(target_list
)
871 for name
in sorted(totals
):
874 # Display the average difference in this name for this
876 avg_diff
= float(diff
) / count
877 color
= self
.col
.RED
if avg_diff
> 0 else self
.col
.GREEN
878 msg
= ' %s %+1.1f' % (name
, avg_diff
)
880 Print('%10s: (for %d/%d boards)' % (arch
, count
,
881 arch_count
[arch
]), newline
=False)
883 Print(msg
, colour
=color
, newline
=False)
888 self
.PrintSizeDetail(target_list
, show_bloat
)
891 def PrintResultSummary(self
, board_selected
, board_dict
, err_lines
,
892 err_line_boards
, warn_lines
, warn_line_boards
,
893 show_sizes
, show_detail
, show_bloat
):
894 """Compare results with the base results and display delta.
896 Only boards mentioned in board_selected will be considered. This
897 function is intended to be called repeatedly with the results of
898 each commit. It therefore shows a 'diff' between what it saw in
899 the last call and what it sees now.
902 board_selected: Dict containing boards to summarise, keyed by
904 board_dict: Dict containing boards for which we built this
905 commit, keyed by board.target. The value is an Outcome object.
906 err_lines: A list of errors for this commit, or [] if there is
907 none, or we don't want to print errors
908 err_line_boards: Dict keyed by error line, containing a list of
909 the Board objects with that error
910 warn_lines: A list of warnings for this commit, or [] if there is
911 none, or we don't want to print errors
912 warn_line_boards: Dict keyed by warning line, containing a list of
913 the Board objects with that warning
914 show_sizes: Show image size deltas
915 show_detail: Show detail for each board
916 show_bloat: Show detail for each function
918 def _BoardList(line
, line_boards
):
919 """Helper function to get a line of boards containing a line
922 line: Error line to search for
924 String containing a list of boards with that error line, or
925 '' if the user has not requested such a list
927 if self
._list
_error
_boards
:
929 for board
in line_boards
[line
]:
930 if not board
.target
in names
:
931 names
.append(board
.target
)
932 names_str
= '(%s) ' % ','.join(names
)
937 def _CalcErrorDelta(base_lines
, base_line_boards
, lines
, line_boards
,
942 if line
not in base_lines
:
943 worse_lines
.append(char
+ '+' +
944 _BoardList(line
, line_boards
) + line
)
945 for line
in base_lines
:
946 if line
not in lines
:
947 better_lines
.append(char
+ '-' +
948 _BoardList(line
, base_line_boards
) + line
)
949 return better_lines
, worse_lines
951 better
= [] # List of boards fixed since last commit
952 worse
= [] # List of new broken boards since last commit
953 new
= [] # List of boards that didn't exist last time
954 unknown
= [] # List of boards that were not built
956 for target
in board_dict
:
957 if target
not in board_selected
:
960 # If the board was built last time, add its outcome to a list
961 if target
in self
._base
_board
_dict
:
962 base_outcome
= self
._base
_board
_dict
[target
].rc
963 outcome
= board_dict
[target
]
964 if outcome
.rc
== OUTCOME_UNKNOWN
:
965 unknown
.append(target
)
966 elif outcome
.rc
< base_outcome
:
967 better
.append(target
)
968 elif outcome
.rc
> base_outcome
:
973 # Get a list of errors that have appeared, and disappeared
974 better_err
, worse_err
= _CalcErrorDelta(self
._base
_err
_lines
,
975 self
._base
_err
_line
_boards
, err_lines
, err_line_boards
, '')
976 better_warn
, worse_warn
= _CalcErrorDelta(self
._base
_warn
_lines
,
977 self
._base
_warn
_line
_boards
, warn_lines
, warn_line_boards
, 'w')
979 # Display results by arch
980 if (better
or worse
or unknown
or new
or worse_err
or better_err
981 or worse_warn
or better_warn
):
983 self
.AddOutcome(board_selected
, arch_list
, better
, '',
985 self
.AddOutcome(board_selected
, arch_list
, worse
, '+',
987 self
.AddOutcome(board_selected
, arch_list
, new
, '*', self
.col
.BLUE
)
988 if self
._show
_unknown
:
989 self
.AddOutcome(board_selected
, arch_list
, unknown
, '?',
991 for arch
, target_list
in arch_list
.iteritems():
992 Print('%10s: %s' % (arch
, target_list
))
993 self
._error
_lines
+= 1
995 Print('\n'.join(better_err
), colour
=self
.col
.GREEN
)
996 self
._error
_lines
+= 1
998 Print('\n'.join(worse_err
), colour
=self
.col
.RED
)
999 self
._error
_lines
+= 1
1001 Print('\n'.join(better_warn
), colour
=self
.col
.CYAN
)
1002 self
._error
_lines
+= 1
1004 Print('\n'.join(worse_warn
), colour
=self
.col
.MAGENTA
)
1005 self
._error
_lines
+= 1
1008 self
.PrintSizeSummary(board_selected
, board_dict
, show_detail
,
1011 # Save our updated information for the next call to this function
1012 self
._base
_board
_dict
= board_dict
1013 self
._base
_err
_lines
= err_lines
1014 self
._base
_warn
_lines
= warn_lines
1015 self
._base
_err
_line
_boards
= err_line_boards
1016 self
._base
_warn
_line
_boards
= warn_line_boards
1018 # Get a list of boards that did not get built, if needed
1020 for board
in board_selected
:
1021 if not board
in board_dict
:
1022 not_built
.append(board
)
1024 Print("Boards not built (%d): %s" % (len(not_built
),
1025 ', '.join(not_built
)))
1027 def ProduceResultSummary(self
, commit_upto
, commits
, board_selected
):
1028 (board_dict
, err_lines
, err_line_boards
, warn_lines
,
1029 warn_line_boards
) = self
.GetResultSummary(
1030 board_selected
, commit_upto
,
1031 read_func_sizes
=self
._show
_bloat
)
1033 msg
= '%02d: %s' % (commit_upto
+ 1,
1034 commits
[commit_upto
].subject
)
1035 Print(msg
, colour
=self
.col
.BLUE
)
1036 self
.PrintResultSummary(board_selected
, board_dict
,
1037 err_lines
if self
._show
_errors
else [], err_line_boards
,
1038 warn_lines
if self
._show
_errors
else [], warn_line_boards
,
1039 self
._show
_sizes
, self
._show
_detail
, self
._show
_bloat
)
1041 def ShowSummary(self
, commits
, board_selected
):
1042 """Show a build summary for U-Boot for a given board list.
1044 Reset the result summary, then repeatedly call GetResultSummary on
1045 each commit's results, then display the differences we see.
1048 commit: Commit objects to summarise
1049 board_selected: Dict containing boards to summarise
1051 self
.commit_count
= len(commits
) if commits
else 1
1052 self
.commits
= commits
1053 self
.ResetResultSummary(board_selected
)
1054 self
._error
_lines
= 0
1056 for commit_upto
in range(0, self
.commit_count
, self
._step
):
1057 self
.ProduceResultSummary(commit_upto
, commits
, board_selected
)
1058 if not self
._error
_lines
:
1059 Print('(no errors to report)', colour
=self
.col
.GREEN
)
1062 def SetupBuild(self
, board_selected
, commits
):
1063 """Set up ready to start a build.
1066 board_selected: Selected boards to build
1067 commits: Selected commits to build
1069 # First work out how many commits we will build
1070 count
= (self
.commit_count
+ self
._step
- 1) / self
._step
1071 self
.count
= len(board_selected
) * count
1072 self
.upto
= self
.warned
= self
.fail
= 0
1073 self
._timestamps
= collections
.deque()
1075 def GetThreadDir(self
, thread_num
):
1076 """Get the directory path to the working dir for a thread.
1079 thread_num: Number of thread to check.
1081 return os
.path
.join(self
._working
_dir
, '%02d' % thread_num
)
1083 def _PrepareThread(self
, thread_num
, setup_git
):
1084 """Prepare the working directory for a thread.
1086 This clones or fetches the repo into the thread's work directory.
1089 thread_num: Thread number (0, 1, ...)
1090 setup_git: True to set up a git repo clone
1092 thread_dir
= self
.GetThreadDir(thread_num
)
1093 builderthread
.Mkdir(thread_dir
)
1094 git_dir
= os
.path
.join(thread_dir
, '.git')
1096 # Clone the repo if it doesn't already exist
1097 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1098 # we have a private index but uses the origin repo's contents?
1099 if setup_git
and self
.git_dir
:
1100 src_dir
= os
.path
.abspath(self
.git_dir
)
1101 if os
.path
.exists(git_dir
):
1102 gitutil
.Fetch(git_dir
, thread_dir
)
1104 Print('Cloning repo for thread %d' % thread_num
)
1105 gitutil
.Clone(src_dir
, thread_dir
)
1107 def _PrepareWorkingSpace(self
, max_threads
, setup_git
):
1108 """Prepare the working directory for use.
1110 Set up the git repo for each thread.
1113 max_threads: Maximum number of threads we expect to need.
1114 setup_git: True to set up a git repo clone
1116 builderthread
.Mkdir(self
._working
_dir
)
1117 for thread
in range(max_threads
):
1118 self
._PrepareThread
(thread
, setup_git
)
1120 def _PrepareOutputSpace(self
):
1121 """Get the output directories ready to receive files.
1123 We delete any output directories which look like ones we need to
1124 create. Having left over directories is confusing when the user wants
1125 to check the output manually.
1127 if not self
.commits
:
1130 for commit_upto
in range(self
.commit_count
):
1131 dir_list
.append(self
._GetOutputDir
(commit_upto
))
1133 for dirname
in glob
.glob(os
.path
.join(self
.base_dir
, '*')):
1134 if dirname
not in dir_list
:
1135 shutil
.rmtree(dirname
)
1137 def BuildBoards(self
, commits
, board_selected
, keep_outputs
, verbose
):
1138 """Build all commits for a list of boards
1141 commits: List of commits to be build, each a Commit object
1142 boards_selected: Dict of selected boards, key is target name,
1143 value is Board object
1144 keep_outputs: True to save build output files
1145 verbose: Display build results as they are completed
1148 - number of boards that failed to build
1149 - number of boards that issued warnings
1151 self
.commit_count
= len(commits
) if commits
else 1
1152 self
.commits
= commits
1153 self
._verbose
= verbose
1155 self
.ResetResultSummary(board_selected
)
1156 builderthread
.Mkdir(self
.base_dir
, parents
= True)
1157 self
._PrepareWorkingSpace
(min(self
.num_threads
, len(board_selected
)),
1158 commits
is not None)
1159 self
._PrepareOutputSpace
()
1160 self
.SetupBuild(board_selected
, commits
)
1161 self
.ProcessResult(None)
1163 # Create jobs to build all commits for each board
1164 for brd
in board_selected
.itervalues():
1165 job
= builderthread
.BuilderJob()
1167 job
.commits
= commits
1168 job
.keep_outputs
= keep_outputs
1169 job
.step
= self
._step
1172 # Wait until all jobs are started
1175 # Wait until we have processed all output
1176 self
.out_queue
.join()
1179 return (self
.fail
, self
.warned
)