blob: 7002034221d12789473d33e7f62cb21cbe296be7 [file] [log] [blame]
Simon Glassc05694f2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
4#
Wolfgang Denkd79de1d2013-07-08 09:37:19 +02005# SPDX-License-Identifier: GPL-2.0+
Simon Glassc05694f2013-04-03 11:07:16 +00006#
7
8import collections
Simon Glassc05694f2013-04-03 11:07:16 +00009from datetime import datetime, timedelta
10import glob
11import os
12import re
13import Queue
14import shutil
15import string
16import sys
Simon Glassc05694f2013-04-03 11:07:16 +000017import time
18
Simon Glass4a1e88b2014-08-09 15:33:00 -060019import builderthread
Simon Glassc05694f2013-04-03 11:07:16 +000020import command
21import gitutil
22import terminal
Simon Glass4433aa92014-09-05 19:00:07 -060023from terminal import Print
Simon Glassc05694f2013-04-03 11:07:16 +000024import toolchain
25
26
27"""
28Theory of Operation
29
30Please see README for user documentation, and you should be familiar with
31that before trying to make sense of this.
32
33Buildman works by keeping the machine as busy as possible, building different
34commits for different boards on multiple CPUs at once.
35
36The source repo (self.git_dir) contains all the commits to be built. Each
37thread works on a single board at a time. It checks out the first commit,
38configures it for that board, then builds it. Then it checks out the next
39commit and builds it (typically without re-configuring). When it runs out
40of commits, it gets another job from the builder and starts again with that
41board.
42
43Clearly the builder threads could work either way - they could check out a
44commit and then built it for all boards. Using separate directories for each
45commit/board pair they could leave their build product around afterwards
46also.
47
48The intent behind building a single board for multiple commits, is to make
49use of incremental builds. Since each commit is built incrementally from
50the previous one, builds are faster. Reconfiguring for a different board
51removes all intermediate object files.
52
53Many threads can be working at once, but each has its own working directory.
54When a thread finishes a build, it puts the output files into a result
55directory.
56
57The base directory used by buildman is normally '../<branch>', i.e.
58a directory higher than the source repository and named after the branch
59being built.
60
61Within the base directory, we have one subdirectory for each commit. Within
62that is one subdirectory for each board. Within that is the build output for
63that commit/board combination.
64
65Buildman also create working directories for each thread, in a .bm-work/
66subdirectory in the base dir.
67
68As an example, say we are building branch 'us-net' for boards 'sandbox' and
69'seaboard', and say that us-net has two commits. We will have directories
70like this:
71
72us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
74 sandbox/
75 u-boot.bin
76 seaboard/
77 u-boot.bin
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
79 sandbox/
80 u-boot.bin
81 seaboard/
82 u-boot.bin
83 .bm-work/
84 00/ working directory for thread 0 (contains source checkout)
85 build/ build output
86 01/ working directory for thread 1
87 build/ build output
88 ...
89u-boot/ source directory
90 .git/ repository
91"""
92
93# Possible build outcomes
94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
95
96# Translate a commit subject into a valid filename
97trans_valid_chars = string.maketrans("/: ", "---")
98
99
Simon Glassc05694f2013-04-03 11:07:16 +0000100class Builder:
101 """Class for building U-Boot for a particular commit.
102
103 Public members: (many should ->private)
104 active: True if the builder is active and has not been stopped
105 already_done: Number of builds already completed
106 base_dir: Base directory to use for builder
107 checkout: True to check out source, False to skip that step.
108 This is used for testing.
109 col: terminal.Color() object
110 count: Number of commits to build
111 do_make: Method to call to invoke Make
112 fail: Number of builds that failed due to error
113 force_build: Force building even if a build already exists
114 force_config_on_failure: If a commit fails for a board, disable
115 incremental building for the next commit we build for that
116 board, so that we will see all warnings/errors again.
Simon Glass7041c392014-07-13 12:22:31 -0600117 force_build_failures: If a previously-built build (i.e. built on
118 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassc05694f2013-04-03 11:07:16 +0000119 git_dir: Git directory containing source repository
120 last_line_len: Length of the last line we printed (used for erasing
121 it with new progress information)
122 num_jobs: Number of jobs to run at once (passed to make as -j)
123 num_threads: Number of builder threads to run
124 out_queue: Queue of results to process
125 re_make_err: Compiled regular expression for ignore_lines
126 queue: Queue of jobs to run
127 threads: List of active threads
128 toolchains: Toolchains object to use for building
129 upto: Current commit number we are building (0.count-1)
130 warned: Number of builds that produced at least one warning
Simon Glassf3018b7a2014-07-14 17:51:02 -0600131 force_reconfig: Reconfigure U-Boot on each comiit. This disables
132 incremental building, where buildman reconfigures on the first
133 commit for a baord, and then just does an incremental build for
134 the following commits. In fact buildman will reconfigure and
135 retry for any failing commits, so generally the only effect of
136 this option is to slow things down.
Simon Glass38df2e22014-07-14 17:51:03 -0600137 in_tree: Build U-Boot in-tree instead of specifying an output
138 directory separate from the source code. This option is really
139 only useful for testing in-tree builds.
Simon Glassc05694f2013-04-03 11:07:16 +0000140
141 Private members:
142 _base_board_dict: Last-summarised Dict of boards
143 _base_err_lines: Last-summarised list of errors
Simon Glass03749d42014-08-28 09:43:44 -0600144 _base_warn_lines: Last-summarised list of warnings
Simon Glassc05694f2013-04-03 11:07:16 +0000145 _build_period_us: Time taken for a single build (float object).
146 _complete_delay: Expected delay until completion (timedelta)
147 _next_delay_update: Next time we plan to display a progress update
148 (datatime)
149 _show_unknown: Show unknown boards (those not built) in summary
150 _timestamps: List of timestamps for the completion of the last
151 last _timestamp_count builds. Each is a datetime object.
152 _timestamp_count: Number of timestamps to keep in our list.
153 _working_dir: Base working directory containing all threads
154 """
155 class Outcome:
156 """Records a build outcome for a single make invocation
157
158 Public Members:
159 rc: Outcome value (OUTCOME_...)
160 err_lines: List of error lines or [] if none
161 sizes: Dictionary of image size information, keyed by filename
162 - Each value is itself a dictionary containing
163 values for 'text', 'data' and 'bss', being the integer
164 size in bytes of each section.
165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
166 value is itself a dictionary:
167 key: function name
168 value: Size of function in bytes
169 """
170 def __init__(self, rc, err_lines, sizes, func_sizes):
171 self.rc = rc
172 self.err_lines = err_lines
173 self.sizes = sizes
174 self.func_sizes = func_sizes
175
176 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900177 gnu_make='make', checkout=True, show_unknown=True, step=1):
Simon Glassc05694f2013-04-03 11:07:16 +0000178 """Create a new Builder object
179
180 Args:
181 toolchains: Toolchains object to use for building
182 base_dir: Base directory to use for builder
183 git_dir: Git directory containing source repository
184 num_threads: Number of builder threads to run
185 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900186 gnu_make: the command name of GNU Make.
Simon Glassc05694f2013-04-03 11:07:16 +0000187 checkout: True to check out source, False to skip that step.
188 This is used for testing.
189 show_unknown: Show unknown boards (those not built) in summary
190 step: 1 to process every commit, n to process every nth commit
191 """
192 self.toolchains = toolchains
193 self.base_dir = base_dir
194 self._working_dir = os.path.join(base_dir, '.bm-work')
195 self.threads = []
196 self.active = True
197 self.do_make = self.Make
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900198 self.gnu_make = gnu_make
Simon Glassc05694f2013-04-03 11:07:16 +0000199 self.checkout = checkout
200 self.num_threads = num_threads
201 self.num_jobs = num_jobs
202 self.already_done = 0
203 self.force_build = False
204 self.git_dir = git_dir
205 self._show_unknown = show_unknown
206 self._timestamp_count = 10
207 self._build_period_us = None
208 self._complete_delay = None
209 self._next_delay_update = datetime.now()
210 self.force_config_on_failure = True
Simon Glass7041c392014-07-13 12:22:31 -0600211 self.force_build_failures = False
Simon Glassf3018b7a2014-07-14 17:51:02 -0600212 self.force_reconfig = False
Simon Glassc05694f2013-04-03 11:07:16 +0000213 self._step = step
Simon Glass38df2e22014-07-14 17:51:03 -0600214 self.in_tree = False
Simon Glassbb4dffb2014-08-09 15:33:06 -0600215 self._error_lines = 0
Simon Glassc05694f2013-04-03 11:07:16 +0000216
217 self.col = terminal.Color()
218
Simon Glass03749d42014-08-28 09:43:44 -0600219 self._re_function = re.compile('(.*): In function.*')
220 self._re_files = re.compile('In file included from.*')
221 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
222 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
223
Simon Glassc05694f2013-04-03 11:07:16 +0000224 self.queue = Queue.Queue()
225 self.out_queue = Queue.Queue()
226 for i in range(self.num_threads):
Simon Glass4a1e88b2014-08-09 15:33:00 -0600227 t = builderthread.BuilderThread(self, i)
Simon Glassc05694f2013-04-03 11:07:16 +0000228 t.setDaemon(True)
229 t.start()
230 self.threads.append(t)
231
232 self.last_line_len = 0
Simon Glass4a1e88b2014-08-09 15:33:00 -0600233 t = builderthread.ResultThread(self)
Simon Glassc05694f2013-04-03 11:07:16 +0000234 t.setDaemon(True)
235 t.start()
236 self.threads.append(t)
237
238 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
239 self.re_make_err = re.compile('|'.join(ignore_lines))
240
241 def __del__(self):
242 """Get rid of all threads created by the builder"""
243 for t in self.threads:
244 del t
245
Simon Glasseb48bbc2014-08-09 15:33:02 -0600246 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glass3394c9f2014-08-28 09:43:43 -0600247 show_detail=False, show_bloat=False,
248 list_error_boards=False):
Simon Glasseb48bbc2014-08-09 15:33:02 -0600249 """Setup display options for the builder.
250
251 show_errors: True to show summarised error/warning info
252 show_sizes: Show size deltas
253 show_detail: Show detail for each board
254 show_bloat: Show detail for each function
Simon Glass3394c9f2014-08-28 09:43:43 -0600255 list_error_boards: Show the boards which caused each error/warning
Simon Glasseb48bbc2014-08-09 15:33:02 -0600256 """
257 self._show_errors = show_errors
258 self._show_sizes = show_sizes
259 self._show_detail = show_detail
260 self._show_bloat = show_bloat
Simon Glass3394c9f2014-08-28 09:43:43 -0600261 self._list_error_boards = list_error_boards
Simon Glasseb48bbc2014-08-09 15:33:02 -0600262
Simon Glassc05694f2013-04-03 11:07:16 +0000263 def _AddTimestamp(self):
264 """Add a new timestamp to the list and record the build period.
265
266 The build period is the length of time taken to perform a single
267 build (one board, one commit).
268 """
269 now = datetime.now()
270 self._timestamps.append(now)
271 count = len(self._timestamps)
272 delta = self._timestamps[-1] - self._timestamps[0]
273 seconds = delta.total_seconds()
274
275 # If we have enough data, estimate build period (time taken for a
276 # single build) and therefore completion time.
277 if count > 1 and self._next_delay_update < now:
278 self._next_delay_update = now + timedelta(seconds=2)
279 if seconds > 0:
280 self._build_period = float(seconds) / count
281 todo = self.count - self.upto
282 self._complete_delay = timedelta(microseconds=
283 self._build_period * todo * 1000000)
284 # Round it
285 self._complete_delay -= timedelta(
286 microseconds=self._complete_delay.microseconds)
287
288 if seconds > 60:
289 self._timestamps.popleft()
290 count -= 1
291
292 def ClearLine(self, length):
293 """Clear any characters on the current line
294
295 Make way for a new line of length 'length', by outputting enough
296 spaces to clear out the old line. Then remember the new length for
297 next time.
298
299 Args:
300 length: Length of new line, in characters
301 """
302 if length < self.last_line_len:
Simon Glass4433aa92014-09-05 19:00:07 -0600303 Print(' ' * (self.last_line_len - length), newline=False)
304 Print('\r', newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000305 self.last_line_len = length
306 sys.stdout.flush()
307
308 def SelectCommit(self, commit, checkout=True):
309 """Checkout the selected commit for this build
310 """
311 self.commit = commit
312 if checkout and self.checkout:
313 gitutil.Checkout(commit.hash)
314
315 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
316 """Run make
317
318 Args:
319 commit: Commit object that is being built
320 brd: Board object that is being built
Roger Meiere0a0e552014-08-20 22:10:29 +0200321 stage: Stage that we are at (mrproper, config, build)
Simon Glassc05694f2013-04-03 11:07:16 +0000322 cwd: Directory where make should be run
323 args: Arguments to pass to make
324 kwargs: Arguments to pass to command.RunPipe()
325 """
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900326 cmd = [self.gnu_make] + list(args)
Simon Glassc05694f2013-04-03 11:07:16 +0000327 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
328 cwd=cwd, raise_on_error=False, **kwargs)
329 return result
330
331 def ProcessResult(self, result):
332 """Process the result of a build, showing progress information
333
334 Args:
Simon Glass78e418e2014-08-09 15:33:03 -0600335 result: A CommandResult object, which indicates the result for
336 a single build
Simon Glassc05694f2013-04-03 11:07:16 +0000337 """
338 col = terminal.Color()
339 if result:
340 target = result.brd.target
341
342 if result.return_code < 0:
343 self.active = False
344 command.StopAll()
345 return
346
347 self.upto += 1
348 if result.return_code != 0:
349 self.fail += 1
350 elif result.stderr:
351 self.warned += 1
352 if result.already_done:
353 self.already_done += 1
Simon Glass78e418e2014-08-09 15:33:03 -0600354 if self._verbose:
Simon Glass4433aa92014-09-05 19:00:07 -0600355 Print('\r', newline=False)
Simon Glass78e418e2014-08-09 15:33:03 -0600356 self.ClearLine(0)
357 boards_selected = {target : result.brd}
358 self.ResetResultSummary(boards_selected)
359 self.ProduceResultSummary(result.commit_upto, self.commits,
360 boards_selected)
Simon Glassc05694f2013-04-03 11:07:16 +0000361 else:
362 target = '(starting)'
363
364 # Display separate counts for ok, warned and fail
365 ok = self.upto - self.warned - self.fail
366 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
367 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
368 line += self.col.Color(self.col.RED, '%5d' % self.fail)
369
370 name = ' /%-5d ' % self.count
371
372 # Add our current completion time estimate
373 self._AddTimestamp()
374 if self._complete_delay:
375 name += '%s : ' % self._complete_delay
376 # When building all boards for a commit, we can print a commit
377 # progress message.
378 if result and result.commit_upto is None:
379 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
380 self.commit_count)
381
382 name += target
Simon Glass4433aa92014-09-05 19:00:07 -0600383 Print(line + name, newline=False)
Simon Glass78e418e2014-08-09 15:33:03 -0600384 length = 14 + len(name)
Simon Glassc05694f2013-04-03 11:07:16 +0000385 self.ClearLine(length)
386
387 def _GetOutputDir(self, commit_upto):
388 """Get the name of the output directory for a commit number
389
390 The output directory is typically .../<branch>/<commit>.
391
392 Args:
393 commit_upto: Commit number to use (0..self.count-1)
394 """
Simon Glassd326ad72014-08-09 15:32:59 -0600395 if self.commits:
396 commit = self.commits[commit_upto]
397 subject = commit.subject.translate(trans_valid_chars)
398 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
399 self.commit_count, commit.hash, subject[:20]))
400 else:
401 commit_dir = 'current'
Simon Glassc05694f2013-04-03 11:07:16 +0000402 output_dir = os.path.join(self.base_dir, commit_dir)
403 return output_dir
404
405 def GetBuildDir(self, commit_upto, target):
406 """Get the name of the build directory for a commit number
407
408 The build directory is typically .../<branch>/<commit>/<target>.
409
410 Args:
411 commit_upto: Commit number to use (0..self.count-1)
412 target: Target name
413 """
414 output_dir = self._GetOutputDir(commit_upto)
415 return os.path.join(output_dir, target)
416
417 def GetDoneFile(self, commit_upto, target):
418 """Get the name of the done file for a commit number
419
420 Args:
421 commit_upto: Commit number to use (0..self.count-1)
422 target: Target name
423 """
424 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
425
426 def GetSizesFile(self, commit_upto, target):
427 """Get the name of the sizes file for a commit number
428
429 Args:
430 commit_upto: Commit number to use (0..self.count-1)
431 target: Target name
432 """
433 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
434
435 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
436 """Get the name of the funcsizes file for a commit number and ELF file
437
438 Args:
439 commit_upto: Commit number to use (0..self.count-1)
440 target: Target name
441 elf_fname: Filename of elf image
442 """
443 return os.path.join(self.GetBuildDir(commit_upto, target),
444 '%s.sizes' % elf_fname.replace('/', '-'))
445
446 def GetObjdumpFile(self, commit_upto, target, elf_fname):
447 """Get the name of the objdump file for a commit number and ELF file
448
449 Args:
450 commit_upto: Commit number to use (0..self.count-1)
451 target: Target name
452 elf_fname: Filename of elf image
453 """
454 return os.path.join(self.GetBuildDir(commit_upto, target),
455 '%s.objdump' % elf_fname.replace('/', '-'))
456
457 def GetErrFile(self, commit_upto, target):
458 """Get the name of the err file for a commit number
459
460 Args:
461 commit_upto: Commit number to use (0..self.count-1)
462 target: Target name
463 """
464 output_dir = self.GetBuildDir(commit_upto, target)
465 return os.path.join(output_dir, 'err')
466
467 def FilterErrors(self, lines):
468 """Filter out errors in which we have no interest
469
470 We should probably use map().
471
472 Args:
473 lines: List of error lines, each a string
474 Returns:
475 New list with only interesting lines included
476 """
477 out_lines = []
478 for line in lines:
479 if not self.re_make_err.search(line):
480 out_lines.append(line)
481 return out_lines
482
483 def ReadFuncSizes(self, fname, fd):
484 """Read function sizes from the output of 'nm'
485
486 Args:
487 fd: File containing data to read
488 fname: Filename we are reading from (just for errors)
489
490 Returns:
491 Dictionary containing size of each function in bytes, indexed by
492 function name.
493 """
494 sym = {}
495 for line in fd.readlines():
496 try:
497 size, type, name = line[:-1].split()
498 except:
Simon Glass4433aa92014-09-05 19:00:07 -0600499 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassc05694f2013-04-03 11:07:16 +0000500 continue
501 if type in 'tTdDbB':
502 # function names begin with '.' on 64-bit powerpc
503 if '.' in name[1:]:
504 name = 'static.' + name.split('.')[0]
505 sym[name] = sym.get(name, 0) + int(size, 16)
506 return sym
507
508 def GetBuildOutcome(self, commit_upto, target, read_func_sizes):
509 """Work out the outcome of a build.
510
511 Args:
512 commit_upto: Commit number to check (0..n-1)
513 target: Target board to check
514 read_func_sizes: True to read function size information
515
516 Returns:
517 Outcome object
518 """
519 done_file = self.GetDoneFile(commit_upto, target)
520 sizes_file = self.GetSizesFile(commit_upto, target)
521 sizes = {}
522 func_sizes = {}
523 if os.path.exists(done_file):
524 with open(done_file, 'r') as fd:
525 return_code = int(fd.readline())
526 err_lines = []
527 err_file = self.GetErrFile(commit_upto, target)
528 if os.path.exists(err_file):
529 with open(err_file, 'r') as fd:
530 err_lines = self.FilterErrors(fd.readlines())
531
532 # Decide whether the build was ok, failed or created warnings
533 if return_code:
534 rc = OUTCOME_ERROR
535 elif len(err_lines):
536 rc = OUTCOME_WARNING
537 else:
538 rc = OUTCOME_OK
539
540 # Convert size information to our simple format
541 if os.path.exists(sizes_file):
542 with open(sizes_file, 'r') as fd:
543 for line in fd.readlines():
544 values = line.split()
545 rodata = 0
546 if len(values) > 6:
547 rodata = int(values[6], 16)
548 size_dict = {
549 'all' : int(values[0]) + int(values[1]) +
550 int(values[2]),
551 'text' : int(values[0]) - rodata,
552 'data' : int(values[1]),
553 'bss' : int(values[2]),
554 'rodata' : rodata,
555 }
556 sizes[values[5]] = size_dict
557
558 if read_func_sizes:
559 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
560 for fname in glob.glob(pattern):
561 with open(fname, 'r') as fd:
562 dict_name = os.path.basename(fname).replace('.sizes',
563 '')
564 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
565
566 return Builder.Outcome(rc, err_lines, sizes, func_sizes)
567
568 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {})
569
570 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes):
571 """Calculate a summary of the results of building a commit.
572
573 Args:
574 board_selected: Dict containing boards to summarise
575 commit_upto: Commit number to summarize (0..self.count-1)
576 read_func_sizes: True to read function size information
577
578 Returns:
579 Tuple:
580 Dict containing boards which passed building this commit.
581 keyed by board.target
Simon Glass03749d42014-08-28 09:43:44 -0600582 List containing a summary of error lines
Simon Glass3394c9f2014-08-28 09:43:43 -0600583 Dict keyed by error line, containing a list of the Board
584 objects with that error
Simon Glass03749d42014-08-28 09:43:44 -0600585 List containing a summary of warning lines
586 Dict keyed by error line, containing a list of the Board
587 objects with that warning
Simon Glassc05694f2013-04-03 11:07:16 +0000588 """
Simon Glass03749d42014-08-28 09:43:44 -0600589 def AddLine(lines_summary, lines_boards, line, board):
590 line = line.rstrip()
591 if line in lines_boards:
592 lines_boards[line].append(board)
593 else:
594 lines_boards[line] = [board]
595 lines_summary.append(line)
596
Simon Glassc05694f2013-04-03 11:07:16 +0000597 board_dict = {}
598 err_lines_summary = []
Simon Glass3394c9f2014-08-28 09:43:43 -0600599 err_lines_boards = {}
Simon Glass03749d42014-08-28 09:43:44 -0600600 warn_lines_summary = []
601 warn_lines_boards = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000602
603 for board in boards_selected.itervalues():
604 outcome = self.GetBuildOutcome(commit_upto, board.target,
605 read_func_sizes)
606 board_dict[board.target] = outcome
Simon Glass03749d42014-08-28 09:43:44 -0600607 last_func = None
608 last_was_warning = False
609 for line in outcome.err_lines:
610 if line:
611 if (self._re_function.match(line) or
612 self._re_files.match(line)):
613 last_func = line
Simon Glass3394c9f2014-08-28 09:43:43 -0600614 else:
Simon Glass03749d42014-08-28 09:43:44 -0600615 is_warning = self._re_warning.match(line)
616 is_note = self._re_note.match(line)
617 if is_warning or (last_was_warning and is_note):
618 if last_func:
619 AddLine(warn_lines_summary, warn_lines_boards,
620 last_func, board)
621 AddLine(warn_lines_summary, warn_lines_boards,
622 line, board)
623 else:
624 if last_func:
625 AddLine(err_lines_summary, err_lines_boards,
626 last_func, board)
627 AddLine(err_lines_summary, err_lines_boards,
628 line, board)
629 last_was_warning = is_warning
630 last_func = None
631 return (board_dict, err_lines_summary, err_lines_boards,
632 warn_lines_summary, warn_lines_boards)
Simon Glassc05694f2013-04-03 11:07:16 +0000633
634 def AddOutcome(self, board_dict, arch_list, changes, char, color):
635 """Add an output to our list of outcomes for each architecture
636
637 This simple function adds failing boards (changes) to the
638 relevant architecture string, so we can print the results out
639 sorted by architecture.
640
641 Args:
642 board_dict: Dict containing all boards
643 arch_list: Dict keyed by arch name. Value is a string containing
644 a list of board names which failed for that arch.
645 changes: List of boards to add to arch_list
646 color: terminal.Colour object
647 """
648 done_arch = {}
649 for target in changes:
650 if target in board_dict:
651 arch = board_dict[target].arch
652 else:
653 arch = 'unknown'
654 str = self.col.Color(color, ' ' + target)
655 if not arch in done_arch:
656 str = self.col.Color(color, char) + ' ' + str
657 done_arch[arch] = True
658 if not arch in arch_list:
659 arch_list[arch] = str
660 else:
661 arch_list[arch] += str
662
663
664 def ColourNum(self, num):
665 color = self.col.RED if num > 0 else self.col.GREEN
666 if num == 0:
667 return '0'
668 return self.col.Color(color, str(num))
669
670 def ResetResultSummary(self, board_selected):
671 """Reset the results summary ready for use.
672
673 Set up the base board list to be all those selected, and set the
674 error lines to empty.
675
676 Following this, calls to PrintResultSummary() will use this
677 information to work out what has changed.
678
679 Args:
680 board_selected: Dict containing boards to summarise, keyed by
681 board.target
682 """
683 self._base_board_dict = {}
684 for board in board_selected:
685 self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
686 self._base_err_lines = []
Simon Glass03749d42014-08-28 09:43:44 -0600687 self._base_warn_lines = []
688 self._base_err_line_boards = {}
689 self._base_warn_line_boards = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000690
691 def PrintFuncSizeDetail(self, fname, old, new):
692 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
693 delta, common = [], {}
694
695 for a in old:
696 if a in new:
697 common[a] = 1
698
699 for name in old:
700 if name not in common:
701 remove += 1
702 down += old[name]
703 delta.append([-old[name], name])
704
705 for name in new:
706 if name not in common:
707 add += 1
708 up += new[name]
709 delta.append([new[name], name])
710
711 for name in common:
712 diff = new.get(name, 0) - old.get(name, 0)
713 if diff > 0:
714 grow, up = grow + 1, up + diff
715 elif diff < 0:
716 shrink, down = shrink + 1, down - diff
717 delta.append([diff, name])
718
719 delta.sort()
720 delta.reverse()
721
722 args = [add, -remove, grow, -shrink, up, -down, up - down]
723 if max(args) == 0:
724 return
725 args = [self.ColourNum(x) for x in args]
726 indent = ' ' * 15
Simon Glass4433aa92014-09-05 19:00:07 -0600727 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
728 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
729 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
730 'delta'))
Simon Glassc05694f2013-04-03 11:07:16 +0000731 for diff, name in delta:
732 if diff:
733 color = self.col.RED if diff > 0 else self.col.GREEN
734 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
735 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4433aa92014-09-05 19:00:07 -0600736 Print(msg, colour=color)
Simon Glassc05694f2013-04-03 11:07:16 +0000737
738
739 def PrintSizeDetail(self, target_list, show_bloat):
740 """Show details size information for each board
741
742 Args:
743 target_list: List of targets, each a dict containing:
744 'target': Target name
745 'total_diff': Total difference in bytes across all areas
746 <part_name>: Difference for that part
747 show_bloat: Show detail for each function
748 """
749 targets_by_diff = sorted(target_list, reverse=True,
750 key=lambda x: x['_total_diff'])
751 for result in targets_by_diff:
752 printed_target = False
753 for name in sorted(result):
754 diff = result[name]
755 if name.startswith('_'):
756 continue
757 if diff != 0:
758 color = self.col.RED if diff > 0 else self.col.GREEN
759 msg = ' %s %+d' % (name, diff)
760 if not printed_target:
Simon Glass4433aa92014-09-05 19:00:07 -0600761 Print('%10s %-15s:' % ('', result['_target']),
762 newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000763 printed_target = True
Simon Glass4433aa92014-09-05 19:00:07 -0600764 Print(msg, colour=color, newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000765 if printed_target:
Simon Glass4433aa92014-09-05 19:00:07 -0600766 Print()
Simon Glassc05694f2013-04-03 11:07:16 +0000767 if show_bloat:
768 target = result['_target']
769 outcome = result['_outcome']
770 base_outcome = self._base_board_dict[target]
771 for fname in outcome.func_sizes:
772 self.PrintFuncSizeDetail(fname,
773 base_outcome.func_sizes[fname],
774 outcome.func_sizes[fname])
775
776
777 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
778 show_bloat):
779 """Print a summary of image sizes broken down by section.
780
781 The summary takes the form of one line per architecture. The
782 line contains deltas for each of the sections (+ means the section
783 got bigger, - means smaller). The nunmbers are the average number
784 of bytes that a board in this section increased by.
785
786 For example:
787 powerpc: (622 boards) text -0.0
788 arm: (285 boards) text -0.0
789 nds32: (3 boards) text -8.0
790
791 Args:
792 board_selected: Dict containing boards to summarise, keyed by
793 board.target
794 board_dict: Dict containing boards for which we built this
795 commit, keyed by board.target. The value is an Outcome object.
796 show_detail: Show detail for each board
797 show_bloat: Show detail for each function
798 """
799 arch_list = {}
800 arch_count = {}
801
802 # Calculate changes in size for different image parts
803 # The previous sizes are in Board.sizes, for each board
804 for target in board_dict:
805 if target not in board_selected:
806 continue
807 base_sizes = self._base_board_dict[target].sizes
808 outcome = board_dict[target]
809 sizes = outcome.sizes
810
811 # Loop through the list of images, creating a dict of size
812 # changes for each image/part. We end up with something like
813 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
814 # which means that U-Boot data increased by 5 bytes and SPL
815 # text decreased by 4.
816 err = {'_target' : target}
817 for image in sizes:
818 if image in base_sizes:
819 base_image = base_sizes[image]
820 # Loop through the text, data, bss parts
821 for part in sorted(sizes[image]):
822 diff = sizes[image][part] - base_image[part]
823 col = None
824 if diff:
825 if image == 'u-boot':
826 name = part
827 else:
828 name = image + ':' + part
829 err[name] = diff
830 arch = board_selected[target].arch
831 if not arch in arch_count:
832 arch_count[arch] = 1
833 else:
834 arch_count[arch] += 1
835 if not sizes:
836 pass # Only add to our list when we have some stats
837 elif not arch in arch_list:
838 arch_list[arch] = [err]
839 else:
840 arch_list[arch].append(err)
841
842 # We now have a list of image size changes sorted by arch
843 # Print out a summary of these
844 for arch, target_list in arch_list.iteritems():
845 # Get total difference for each type
846 totals = {}
847 for result in target_list:
848 total = 0
849 for name, diff in result.iteritems():
850 if name.startswith('_'):
851 continue
852 total += diff
853 if name in totals:
854 totals[name] += diff
855 else:
856 totals[name] = diff
857 result['_total_diff'] = total
858 result['_outcome'] = board_dict[result['_target']]
859
860 count = len(target_list)
861 printed_arch = False
862 for name in sorted(totals):
863 diff = totals[name]
864 if diff:
865 # Display the average difference in this name for this
866 # architecture
867 avg_diff = float(diff) / count
868 color = self.col.RED if avg_diff > 0 else self.col.GREEN
869 msg = ' %s %+1.1f' % (name, avg_diff)
870 if not printed_arch:
Simon Glass4433aa92014-09-05 19:00:07 -0600871 Print('%10s: (for %d/%d boards)' % (arch, count,
872 arch_count[arch]), newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000873 printed_arch = True
Simon Glass4433aa92014-09-05 19:00:07 -0600874 Print(msg, colour=color, newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000875
876 if printed_arch:
Simon Glass4433aa92014-09-05 19:00:07 -0600877 Print()
Simon Glassc05694f2013-04-03 11:07:16 +0000878 if show_detail:
879 self.PrintSizeDetail(target_list, show_bloat)
880
881
882 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glass03749d42014-08-28 09:43:44 -0600883 err_line_boards, warn_lines, warn_line_boards,
884 show_sizes, show_detail, show_bloat):
Simon Glassc05694f2013-04-03 11:07:16 +0000885 """Compare results with the base results and display delta.
886
887 Only boards mentioned in board_selected will be considered. This
888 function is intended to be called repeatedly with the results of
889 each commit. It therefore shows a 'diff' between what it saw in
890 the last call and what it sees now.
891
892 Args:
893 board_selected: Dict containing boards to summarise, keyed by
894 board.target
895 board_dict: Dict containing boards for which we built this
896 commit, keyed by board.target. The value is an Outcome object.
897 err_lines: A list of errors for this commit, or [] if there is
898 none, or we don't want to print errors
Simon Glass3394c9f2014-08-28 09:43:43 -0600899 err_line_boards: Dict keyed by error line, containing a list of
900 the Board objects with that error
Simon Glass03749d42014-08-28 09:43:44 -0600901 warn_lines: A list of warnings for this commit, or [] if there is
902 none, or we don't want to print errors
903 warn_line_boards: Dict keyed by warning line, containing a list of
904 the Board objects with that warning
Simon Glassc05694f2013-04-03 11:07:16 +0000905 show_sizes: Show image size deltas
906 show_detail: Show detail for each board
907 show_bloat: Show detail for each function
908 """
Simon Glass03749d42014-08-28 09:43:44 -0600909 def _BoardList(line, line_boards):
Simon Glass3394c9f2014-08-28 09:43:43 -0600910 """Helper function to get a line of boards containing a line
911
912 Args:
913 line: Error line to search for
914 Return:
915 String containing a list of boards with that error line, or
916 '' if the user has not requested such a list
917 """
918 if self._list_error_boards:
919 names = []
Simon Glass03749d42014-08-28 09:43:44 -0600920 for board in line_boards[line]:
Simon Glass85620bb2014-10-16 01:05:55 -0600921 if not board.target in names:
922 names.append(board.target)
Simon Glass3394c9f2014-08-28 09:43:43 -0600923 names_str = '(%s) ' % ','.join(names)
924 else:
925 names_str = ''
926 return names_str
927
Simon Glass03749d42014-08-28 09:43:44 -0600928 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
929 char):
930 better_lines = []
931 worse_lines = []
932 for line in lines:
933 if line not in base_lines:
934 worse_lines.append(char + '+' +
935 _BoardList(line, line_boards) + line)
936 for line in base_lines:
937 if line not in lines:
938 better_lines.append(char + '-' +
939 _BoardList(line, base_line_boards) + line)
940 return better_lines, worse_lines
941
Simon Glassc05694f2013-04-03 11:07:16 +0000942 better = [] # List of boards fixed since last commit
943 worse = [] # List of new broken boards since last commit
944 new = [] # List of boards that didn't exist last time
945 unknown = [] # List of boards that were not built
946
947 for target in board_dict:
948 if target not in board_selected:
949 continue
950
951 # If the board was built last time, add its outcome to a list
952 if target in self._base_board_dict:
953 base_outcome = self._base_board_dict[target].rc
954 outcome = board_dict[target]
955 if outcome.rc == OUTCOME_UNKNOWN:
956 unknown.append(target)
957 elif outcome.rc < base_outcome:
958 better.append(target)
959 elif outcome.rc > base_outcome:
960 worse.append(target)
961 else:
962 new.append(target)
963
964 # Get a list of errors that have appeared, and disappeared
Simon Glass03749d42014-08-28 09:43:44 -0600965 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
966 self._base_err_line_boards, err_lines, err_line_boards, '')
967 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
968 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassc05694f2013-04-03 11:07:16 +0000969
970 # Display results by arch
Simon Glass03749d42014-08-28 09:43:44 -0600971 if (better or worse or unknown or new or worse_err or better_err
972 or worse_warn or better_warn):
Simon Glassc05694f2013-04-03 11:07:16 +0000973 arch_list = {}
974 self.AddOutcome(board_selected, arch_list, better, '',
975 self.col.GREEN)
976 self.AddOutcome(board_selected, arch_list, worse, '+',
977 self.col.RED)
978 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
979 if self._show_unknown:
980 self.AddOutcome(board_selected, arch_list, unknown, '?',
981 self.col.MAGENTA)
982 for arch, target_list in arch_list.iteritems():
Simon Glass4433aa92014-09-05 19:00:07 -0600983 Print('%10s: %s' % (arch, target_list))
Simon Glassbb4dffb2014-08-09 15:33:06 -0600984 self._error_lines += 1
Simon Glassc05694f2013-04-03 11:07:16 +0000985 if better_err:
Simon Glass4433aa92014-09-05 19:00:07 -0600986 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glassbb4dffb2014-08-09 15:33:06 -0600987 self._error_lines += 1
Simon Glassc05694f2013-04-03 11:07:16 +0000988 if worse_err:
Simon Glass4433aa92014-09-05 19:00:07 -0600989 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glassbb4dffb2014-08-09 15:33:06 -0600990 self._error_lines += 1
Simon Glass03749d42014-08-28 09:43:44 -0600991 if better_warn:
Simon Glass4433aa92014-09-05 19:00:07 -0600992 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glass03749d42014-08-28 09:43:44 -0600993 self._error_lines += 1
994 if worse_warn:
Simon Glass4433aa92014-09-05 19:00:07 -0600995 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glass03749d42014-08-28 09:43:44 -0600996 self._error_lines += 1
Simon Glassc05694f2013-04-03 11:07:16 +0000997
998 if show_sizes:
999 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1000 show_bloat)
1001
1002 # Save our updated information for the next call to this function
1003 self._base_board_dict = board_dict
1004 self._base_err_lines = err_lines
Simon Glass03749d42014-08-28 09:43:44 -06001005 self._base_warn_lines = warn_lines
1006 self._base_err_line_boards = err_line_boards
1007 self._base_warn_line_boards = warn_line_boards
Simon Glassc05694f2013-04-03 11:07:16 +00001008
1009 # Get a list of boards that did not get built, if needed
1010 not_built = []
1011 for board in board_selected:
1012 if not board in board_dict:
1013 not_built.append(board)
1014 if not_built:
Simon Glass4433aa92014-09-05 19:00:07 -06001015 Print("Boards not built (%d): %s" % (len(not_built),
1016 ', '.join(not_built)))
Simon Glassc05694f2013-04-03 11:07:16 +00001017
Simon Glasseb48bbc2014-08-09 15:33:02 -06001018 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glass03749d42014-08-28 09:43:44 -06001019 (board_dict, err_lines, err_line_boards, warn_lines,
1020 warn_line_boards) = self.GetResultSummary(
Simon Glass3394c9f2014-08-28 09:43:43 -06001021 board_selected, commit_upto,
1022 read_func_sizes=self._show_bloat)
Simon Glasseb48bbc2014-08-09 15:33:02 -06001023 if commits:
1024 msg = '%02d: %s' % (commit_upto + 1,
1025 commits[commit_upto].subject)
Simon Glass4433aa92014-09-05 19:00:07 -06001026 Print(msg, colour=self.col.BLUE)
Simon Glasseb48bbc2014-08-09 15:33:02 -06001027 self.PrintResultSummary(board_selected, board_dict,
Simon Glass3394c9f2014-08-28 09:43:43 -06001028 err_lines if self._show_errors else [], err_line_boards,
Simon Glass03749d42014-08-28 09:43:44 -06001029 warn_lines if self._show_errors else [], warn_line_boards,
Simon Glasseb48bbc2014-08-09 15:33:02 -06001030 self._show_sizes, self._show_detail, self._show_bloat)
Simon Glassc05694f2013-04-03 11:07:16 +00001031
Simon Glasseb48bbc2014-08-09 15:33:02 -06001032 def ShowSummary(self, commits, board_selected):
Simon Glassc05694f2013-04-03 11:07:16 +00001033 """Show a build summary for U-Boot for a given board list.
1034
1035 Reset the result summary, then repeatedly call GetResultSummary on
1036 each commit's results, then display the differences we see.
1037
1038 Args:
1039 commit: Commit objects to summarise
1040 board_selected: Dict containing boards to summarise
Simon Glassc05694f2013-04-03 11:07:16 +00001041 """
Simon Glassd326ad72014-08-09 15:32:59 -06001042 self.commit_count = len(commits) if commits else 1
Simon Glassc05694f2013-04-03 11:07:16 +00001043 self.commits = commits
1044 self.ResetResultSummary(board_selected)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001045 self._error_lines = 0
Simon Glassc05694f2013-04-03 11:07:16 +00001046
1047 for commit_upto in range(0, self.commit_count, self._step):
Simon Glasseb48bbc2014-08-09 15:33:02 -06001048 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001049 if not self._error_lines:
Simon Glass4433aa92014-09-05 19:00:07 -06001050 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassc05694f2013-04-03 11:07:16 +00001051
1052
1053 def SetupBuild(self, board_selected, commits):
1054 """Set up ready to start a build.
1055
1056 Args:
1057 board_selected: Selected boards to build
1058 commits: Selected commits to build
1059 """
1060 # First work out how many commits we will build
Simon Glassd326ad72014-08-09 15:32:59 -06001061 count = (self.commit_count + self._step - 1) / self._step
Simon Glassc05694f2013-04-03 11:07:16 +00001062 self.count = len(board_selected) * count
1063 self.upto = self.warned = self.fail = 0
1064 self._timestamps = collections.deque()
1065
Simon Glassc05694f2013-04-03 11:07:16 +00001066 def GetThreadDir(self, thread_num):
1067 """Get the directory path to the working dir for a thread.
1068
1069 Args:
1070 thread_num: Number of thread to check.
1071 """
1072 return os.path.join(self._working_dir, '%02d' % thread_num)
1073
Simon Glassd326ad72014-08-09 15:32:59 -06001074 def _PrepareThread(self, thread_num, setup_git):
Simon Glassc05694f2013-04-03 11:07:16 +00001075 """Prepare the working directory for a thread.
1076
1077 This clones or fetches the repo into the thread's work directory.
1078
1079 Args:
1080 thread_num: Thread number (0, 1, ...)
Simon Glassd326ad72014-08-09 15:32:59 -06001081 setup_git: True to set up a git repo clone
Simon Glassc05694f2013-04-03 11:07:16 +00001082 """
1083 thread_dir = self.GetThreadDir(thread_num)
Simon Glass4a1e88b2014-08-09 15:33:00 -06001084 builderthread.Mkdir(thread_dir)
Simon Glassc05694f2013-04-03 11:07:16 +00001085 git_dir = os.path.join(thread_dir, '.git')
1086
1087 # Clone the repo if it doesn't already exist
1088 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1089 # we have a private index but uses the origin repo's contents?
Simon Glassd326ad72014-08-09 15:32:59 -06001090 if setup_git and self.git_dir:
Simon Glassc05694f2013-04-03 11:07:16 +00001091 src_dir = os.path.abspath(self.git_dir)
1092 if os.path.exists(git_dir):
1093 gitutil.Fetch(git_dir, thread_dir)
1094 else:
Simon Glass4433aa92014-09-05 19:00:07 -06001095 Print('Cloning repo for thread %d' % thread_num)
Simon Glassc05694f2013-04-03 11:07:16 +00001096 gitutil.Clone(src_dir, thread_dir)
1097
Simon Glassd326ad72014-08-09 15:32:59 -06001098 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassc05694f2013-04-03 11:07:16 +00001099 """Prepare the working directory for use.
1100
1101 Set up the git repo for each thread.
1102
1103 Args:
1104 max_threads: Maximum number of threads we expect to need.
Simon Glassd326ad72014-08-09 15:32:59 -06001105 setup_git: True to set up a git repo clone
Simon Glassc05694f2013-04-03 11:07:16 +00001106 """
Simon Glass4a1e88b2014-08-09 15:33:00 -06001107 builderthread.Mkdir(self._working_dir)
Simon Glassc05694f2013-04-03 11:07:16 +00001108 for thread in range(max_threads):
Simon Glassd326ad72014-08-09 15:32:59 -06001109 self._PrepareThread(thread, setup_git)
Simon Glassc05694f2013-04-03 11:07:16 +00001110
1111 def _PrepareOutputSpace(self):
1112 """Get the output directories ready to receive files.
1113
1114 We delete any output directories which look like ones we need to
1115 create. Having left over directories is confusing when the user wants
1116 to check the output manually.
1117 """
1118 dir_list = []
1119 for commit_upto in range(self.commit_count):
1120 dir_list.append(self._GetOutputDir(commit_upto))
1121
1122 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1123 if dirname not in dir_list:
1124 shutil.rmtree(dirname)
1125
Simon Glass78e418e2014-08-09 15:33:03 -06001126 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassc05694f2013-04-03 11:07:16 +00001127 """Build all commits for a list of boards
1128
1129 Args:
1130 commits: List of commits to be build, each a Commit object
1131 boards_selected: Dict of selected boards, key is target name,
1132 value is Board object
Simon Glassc05694f2013-04-03 11:07:16 +00001133 keep_outputs: True to save build output files
Simon Glass78e418e2014-08-09 15:33:03 -06001134 verbose: Display build results as they are completed
Simon Glassc2f91072014-08-28 09:43:39 -06001135 Returns:
1136 Tuple containing:
1137 - number of boards that failed to build
1138 - number of boards that issued warnings
Simon Glassc05694f2013-04-03 11:07:16 +00001139 """
Simon Glassd326ad72014-08-09 15:32:59 -06001140 self.commit_count = len(commits) if commits else 1
Simon Glassc05694f2013-04-03 11:07:16 +00001141 self.commits = commits
Simon Glass78e418e2014-08-09 15:33:03 -06001142 self._verbose = verbose
Simon Glassc05694f2013-04-03 11:07:16 +00001143
1144 self.ResetResultSummary(board_selected)
Thierry Reding336d5bd2014-08-19 10:22:39 +02001145 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassd326ad72014-08-09 15:32:59 -06001146 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1147 commits is not None)
Simon Glassc05694f2013-04-03 11:07:16 +00001148 self._PrepareOutputSpace()
1149 self.SetupBuild(board_selected, commits)
1150 self.ProcessResult(None)
1151
1152 # Create jobs to build all commits for each board
1153 for brd in board_selected.itervalues():
Simon Glass4a1e88b2014-08-09 15:33:00 -06001154 job = builderthread.BuilderJob()
Simon Glassc05694f2013-04-03 11:07:16 +00001155 job.board = brd
1156 job.commits = commits
1157 job.keep_outputs = keep_outputs
1158 job.step = self._step
1159 self.queue.put(job)
1160
1161 # Wait until all jobs are started
1162 self.queue.join()
1163
1164 # Wait until we have processed all output
1165 self.out_queue.join()
Simon Glass4433aa92014-09-05 19:00:07 -06001166 Print()
Simon Glassc05694f2013-04-03 11:07:16 +00001167 self.ClearLine(0)
Simon Glassc2f91072014-08-28 09:43:39 -06001168 return (self.fail, self.warned)