blob: c7d3c869688dd7a99cd4f3893059205f94086272 [file] [log] [blame]
Simon Glassc05694f2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
4#
Wolfgang Denkd79de1d2013-07-08 09:37:19 +02005# SPDX-License-Identifier: GPL-2.0+
Simon Glassc05694f2013-04-03 11:07:16 +00006#
7
8import collections
Simon Glassc05694f2013-04-03 11:07:16 +00009from datetime import datetime, timedelta
10import glob
11import os
12import re
13import Queue
14import shutil
15import string
16import sys
Simon Glassc05694f2013-04-03 11:07:16 +000017import time
18
Simon Glass4a1e88b2014-08-09 15:33:00 -060019import builderthread
Simon Glassc05694f2013-04-03 11:07:16 +000020import command
21import gitutil
22import terminal
Simon Glass4433aa92014-09-05 19:00:07 -060023from terminal import Print
Simon Glassc05694f2013-04-03 11:07:16 +000024import toolchain
25
26
27"""
28Theory of Operation
29
30Please see README for user documentation, and you should be familiar with
31that before trying to make sense of this.
32
33Buildman works by keeping the machine as busy as possible, building different
34commits for different boards on multiple CPUs at once.
35
36The source repo (self.git_dir) contains all the commits to be built. Each
37thread works on a single board at a time. It checks out the first commit,
38configures it for that board, then builds it. Then it checks out the next
39commit and builds it (typically without re-configuring). When it runs out
40of commits, it gets another job from the builder and starts again with that
41board.
42
43Clearly the builder threads could work either way - they could check out a
44commit and then built it for all boards. Using separate directories for each
45commit/board pair they could leave their build product around afterwards
46also.
47
48The intent behind building a single board for multiple commits, is to make
49use of incremental builds. Since each commit is built incrementally from
50the previous one, builds are faster. Reconfiguring for a different board
51removes all intermediate object files.
52
53Many threads can be working at once, but each has its own working directory.
54When a thread finishes a build, it puts the output files into a result
55directory.
56
57The base directory used by buildman is normally '../<branch>', i.e.
58a directory higher than the source repository and named after the branch
59being built.
60
61Within the base directory, we have one subdirectory for each commit. Within
62that is one subdirectory for each board. Within that is the build output for
63that commit/board combination.
64
65Buildman also create working directories for each thread, in a .bm-work/
66subdirectory in the base dir.
67
68As an example, say we are building branch 'us-net' for boards 'sandbox' and
69'seaboard', and say that us-net has two commits. We will have directories
70like this:
71
72us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
74 sandbox/
75 u-boot.bin
76 seaboard/
77 u-boot.bin
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
79 sandbox/
80 u-boot.bin
81 seaboard/
82 u-boot.bin
83 .bm-work/
84 00/ working directory for thread 0 (contains source checkout)
85 build/ build output
86 01/ working directory for thread 1
87 build/ build output
88 ...
89u-boot/ source directory
90 .git/ repository
91"""
92
93# Possible build outcomes
94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
95
96# Translate a commit subject into a valid filename
97trans_valid_chars = string.maketrans("/: ", "---")
98
Simon Glassdb17fb82015-02-05 22:06:15 -070099CONFIG_FILENAMES = [
100 '.config', '.config-spl', '.config-tpl',
101 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
102 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
104]
105
Simon Glassc05694f2013-04-03 11:07:16 +0000106
Simon Glassc05694f2013-04-03 11:07:16 +0000107class Builder:
108 """Class for building U-Boot for a particular commit.
109
110 Public members: (many should ->private)
111 active: True if the builder is active and has not been stopped
112 already_done: Number of builds already completed
113 base_dir: Base directory to use for builder
114 checkout: True to check out source, False to skip that step.
115 This is used for testing.
116 col: terminal.Color() object
117 count: Number of commits to build
118 do_make: Method to call to invoke Make
119 fail: Number of builds that failed due to error
120 force_build: Force building even if a build already exists
121 force_config_on_failure: If a commit fails for a board, disable
122 incremental building for the next commit we build for that
123 board, so that we will see all warnings/errors again.
Simon Glass7041c392014-07-13 12:22:31 -0600124 force_build_failures: If a previously-built build (i.e. built on
125 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassc05694f2013-04-03 11:07:16 +0000126 git_dir: Git directory containing source repository
127 last_line_len: Length of the last line we printed (used for erasing
128 it with new progress information)
129 num_jobs: Number of jobs to run at once (passed to make as -j)
130 num_threads: Number of builder threads to run
131 out_queue: Queue of results to process
132 re_make_err: Compiled regular expression for ignore_lines
133 queue: Queue of jobs to run
134 threads: List of active threads
135 toolchains: Toolchains object to use for building
136 upto: Current commit number we are building (0.count-1)
137 warned: Number of builds that produced at least one warning
Simon Glassf3018b7a2014-07-14 17:51:02 -0600138 force_reconfig: Reconfigure U-Boot on each comiit. This disables
139 incremental building, where buildman reconfigures on the first
140 commit for a baord, and then just does an incremental build for
141 the following commits. In fact buildman will reconfigure and
142 retry for any failing commits, so generally the only effect of
143 this option is to slow things down.
Simon Glass38df2e22014-07-14 17:51:03 -0600144 in_tree: Build U-Boot in-tree instead of specifying an output
145 directory separate from the source code. This option is really
146 only useful for testing in-tree builds.
Simon Glassc05694f2013-04-03 11:07:16 +0000147
148 Private members:
149 _base_board_dict: Last-summarised Dict of boards
150 _base_err_lines: Last-summarised list of errors
Simon Glass03749d42014-08-28 09:43:44 -0600151 _base_warn_lines: Last-summarised list of warnings
Simon Glassc05694f2013-04-03 11:07:16 +0000152 _build_period_us: Time taken for a single build (float object).
153 _complete_delay: Expected delay until completion (timedelta)
154 _next_delay_update: Next time we plan to display a progress update
155 (datatime)
156 _show_unknown: Show unknown boards (those not built) in summary
157 _timestamps: List of timestamps for the completion of the last
158 last _timestamp_count builds. Each is a datetime object.
159 _timestamp_count: Number of timestamps to keep in our list.
160 _working_dir: Base working directory containing all threads
161 """
162 class Outcome:
163 """Records a build outcome for a single make invocation
164
165 Public Members:
166 rc: Outcome value (OUTCOME_...)
167 err_lines: List of error lines or [] if none
168 sizes: Dictionary of image size information, keyed by filename
169 - Each value is itself a dictionary containing
170 values for 'text', 'data' and 'bss', being the integer
171 size in bytes of each section.
172 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
173 value is itself a dictionary:
174 key: function name
175 value: Size of function in bytes
Simon Glassdb17fb82015-02-05 22:06:15 -0700176 config: Dictionary keyed by filename - e.g. '.config'. Each
177 value is itself a dictionary:
178 key: config name
179 value: config value
Simon Glassc05694f2013-04-03 11:07:16 +0000180 """
Simon Glassdb17fb82015-02-05 22:06:15 -0700181 def __init__(self, rc, err_lines, sizes, func_sizes, config):
Simon Glassc05694f2013-04-03 11:07:16 +0000182 self.rc = rc
183 self.err_lines = err_lines
184 self.sizes = sizes
185 self.func_sizes = func_sizes
Simon Glassdb17fb82015-02-05 22:06:15 -0700186 self.config = config
Simon Glassc05694f2013-04-03 11:07:16 +0000187
188 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glasse87bde12014-12-01 17:33:55 -0700189 gnu_make='make', checkout=True, show_unknown=True, step=1,
Simon Glass655b6102014-12-01 17:34:07 -0700190 no_subdirs=False, full_path=False, verbose_build=False):
Simon Glassc05694f2013-04-03 11:07:16 +0000191 """Create a new Builder object
192
193 Args:
194 toolchains: Toolchains object to use for building
195 base_dir: Base directory to use for builder
196 git_dir: Git directory containing source repository
197 num_threads: Number of builder threads to run
198 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900199 gnu_make: the command name of GNU Make.
Simon Glassc05694f2013-04-03 11:07:16 +0000200 checkout: True to check out source, False to skip that step.
201 This is used for testing.
202 show_unknown: Show unknown boards (those not built) in summary
203 step: 1 to process every commit, n to process every nth commit
Simon Glassd48a46c2014-12-01 17:34:00 -0700204 no_subdirs: Don't create subdirectories when building current
205 source for a single board
206 full_path: Return the full path in CROSS_COMPILE and don't set
207 PATH
Simon Glass655b6102014-12-01 17:34:07 -0700208 verbose_build: Run build with V=1 and don't use 'make -s'
Simon Glassc05694f2013-04-03 11:07:16 +0000209 """
210 self.toolchains = toolchains
211 self.base_dir = base_dir
212 self._working_dir = os.path.join(base_dir, '.bm-work')
213 self.threads = []
214 self.active = True
215 self.do_make = self.Make
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900216 self.gnu_make = gnu_make
Simon Glassc05694f2013-04-03 11:07:16 +0000217 self.checkout = checkout
218 self.num_threads = num_threads
219 self.num_jobs = num_jobs
220 self.already_done = 0
221 self.force_build = False
222 self.git_dir = git_dir
223 self._show_unknown = show_unknown
224 self._timestamp_count = 10
225 self._build_period_us = None
226 self._complete_delay = None
227 self._next_delay_update = datetime.now()
228 self.force_config_on_failure = True
Simon Glass7041c392014-07-13 12:22:31 -0600229 self.force_build_failures = False
Simon Glassf3018b7a2014-07-14 17:51:02 -0600230 self.force_reconfig = False
Simon Glassc05694f2013-04-03 11:07:16 +0000231 self._step = step
Simon Glass38df2e22014-07-14 17:51:03 -0600232 self.in_tree = False
Simon Glassbb4dffb2014-08-09 15:33:06 -0600233 self._error_lines = 0
Simon Glasse87bde12014-12-01 17:33:55 -0700234 self.no_subdirs = no_subdirs
Simon Glassd48a46c2014-12-01 17:34:00 -0700235 self.full_path = full_path
Simon Glass655b6102014-12-01 17:34:07 -0700236 self.verbose_build = verbose_build
Simon Glassc05694f2013-04-03 11:07:16 +0000237
238 self.col = terminal.Color()
239
Simon Glass03749d42014-08-28 09:43:44 -0600240 self._re_function = re.compile('(.*): In function.*')
241 self._re_files = re.compile('In file included from.*')
242 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
243 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
244
Simon Glassc05694f2013-04-03 11:07:16 +0000245 self.queue = Queue.Queue()
246 self.out_queue = Queue.Queue()
247 for i in range(self.num_threads):
Simon Glass4a1e88b2014-08-09 15:33:00 -0600248 t = builderthread.BuilderThread(self, i)
Simon Glassc05694f2013-04-03 11:07:16 +0000249 t.setDaemon(True)
250 t.start()
251 self.threads.append(t)
252
253 self.last_line_len = 0
Simon Glass4a1e88b2014-08-09 15:33:00 -0600254 t = builderthread.ResultThread(self)
Simon Glassc05694f2013-04-03 11:07:16 +0000255 t.setDaemon(True)
256 t.start()
257 self.threads.append(t)
258
259 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
260 self.re_make_err = re.compile('|'.join(ignore_lines))
261
262 def __del__(self):
263 """Get rid of all threads created by the builder"""
264 for t in self.threads:
265 del t
266
Simon Glasseb48bbc2014-08-09 15:33:02 -0600267 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glass3394c9f2014-08-28 09:43:43 -0600268 show_detail=False, show_bloat=False,
Simon Glassdb17fb82015-02-05 22:06:15 -0700269 list_error_boards=False, show_config=False):
Simon Glasseb48bbc2014-08-09 15:33:02 -0600270 """Setup display options for the builder.
271
272 show_errors: True to show summarised error/warning info
273 show_sizes: Show size deltas
274 show_detail: Show detail for each board
275 show_bloat: Show detail for each function
Simon Glass3394c9f2014-08-28 09:43:43 -0600276 list_error_boards: Show the boards which caused each error/warning
Simon Glassdb17fb82015-02-05 22:06:15 -0700277 show_config: Show config deltas
Simon Glasseb48bbc2014-08-09 15:33:02 -0600278 """
279 self._show_errors = show_errors
280 self._show_sizes = show_sizes
281 self._show_detail = show_detail
282 self._show_bloat = show_bloat
Simon Glass3394c9f2014-08-28 09:43:43 -0600283 self._list_error_boards = list_error_boards
Simon Glassdb17fb82015-02-05 22:06:15 -0700284 self._show_config = show_config
Simon Glasseb48bbc2014-08-09 15:33:02 -0600285
Simon Glassc05694f2013-04-03 11:07:16 +0000286 def _AddTimestamp(self):
287 """Add a new timestamp to the list and record the build period.
288
289 The build period is the length of time taken to perform a single
290 build (one board, one commit).
291 """
292 now = datetime.now()
293 self._timestamps.append(now)
294 count = len(self._timestamps)
295 delta = self._timestamps[-1] - self._timestamps[0]
296 seconds = delta.total_seconds()
297
298 # If we have enough data, estimate build period (time taken for a
299 # single build) and therefore completion time.
300 if count > 1 and self._next_delay_update < now:
301 self._next_delay_update = now + timedelta(seconds=2)
302 if seconds > 0:
303 self._build_period = float(seconds) / count
304 todo = self.count - self.upto
305 self._complete_delay = timedelta(microseconds=
306 self._build_period * todo * 1000000)
307 # Round it
308 self._complete_delay -= timedelta(
309 microseconds=self._complete_delay.microseconds)
310
311 if seconds > 60:
312 self._timestamps.popleft()
313 count -= 1
314
315 def ClearLine(self, length):
316 """Clear any characters on the current line
317
318 Make way for a new line of length 'length', by outputting enough
319 spaces to clear out the old line. Then remember the new length for
320 next time.
321
322 Args:
323 length: Length of new line, in characters
324 """
325 if length < self.last_line_len:
Simon Glass4433aa92014-09-05 19:00:07 -0600326 Print(' ' * (self.last_line_len - length), newline=False)
327 Print('\r', newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000328 self.last_line_len = length
329 sys.stdout.flush()
330
331 def SelectCommit(self, commit, checkout=True):
332 """Checkout the selected commit for this build
333 """
334 self.commit = commit
335 if checkout and self.checkout:
336 gitutil.Checkout(commit.hash)
337
338 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
339 """Run make
340
341 Args:
342 commit: Commit object that is being built
343 brd: Board object that is being built
Roger Meiere0a0e552014-08-20 22:10:29 +0200344 stage: Stage that we are at (mrproper, config, build)
Simon Glassc05694f2013-04-03 11:07:16 +0000345 cwd: Directory where make should be run
346 args: Arguments to pass to make
347 kwargs: Arguments to pass to command.RunPipe()
348 """
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900349 cmd = [self.gnu_make] + list(args)
Simon Glassc05694f2013-04-03 11:07:16 +0000350 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
351 cwd=cwd, raise_on_error=False, **kwargs)
Simon Glass413f91a2015-02-05 22:06:12 -0700352 if self.verbose_build:
353 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
354 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassc05694f2013-04-03 11:07:16 +0000355 return result
356
357 def ProcessResult(self, result):
358 """Process the result of a build, showing progress information
359
360 Args:
Simon Glass78e418e2014-08-09 15:33:03 -0600361 result: A CommandResult object, which indicates the result for
362 a single build
Simon Glassc05694f2013-04-03 11:07:16 +0000363 """
364 col = terminal.Color()
365 if result:
366 target = result.brd.target
367
368 if result.return_code < 0:
369 self.active = False
370 command.StopAll()
371 return
372
373 self.upto += 1
374 if result.return_code != 0:
375 self.fail += 1
376 elif result.stderr:
377 self.warned += 1
378 if result.already_done:
379 self.already_done += 1
Simon Glass78e418e2014-08-09 15:33:03 -0600380 if self._verbose:
Simon Glass4433aa92014-09-05 19:00:07 -0600381 Print('\r', newline=False)
Simon Glass78e418e2014-08-09 15:33:03 -0600382 self.ClearLine(0)
383 boards_selected = {target : result.brd}
384 self.ResetResultSummary(boards_selected)
385 self.ProduceResultSummary(result.commit_upto, self.commits,
386 boards_selected)
Simon Glassc05694f2013-04-03 11:07:16 +0000387 else:
388 target = '(starting)'
389
390 # Display separate counts for ok, warned and fail
391 ok = self.upto - self.warned - self.fail
392 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
393 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
394 line += self.col.Color(self.col.RED, '%5d' % self.fail)
395
396 name = ' /%-5d ' % self.count
397
398 # Add our current completion time estimate
399 self._AddTimestamp()
400 if self._complete_delay:
401 name += '%s : ' % self._complete_delay
402 # When building all boards for a commit, we can print a commit
403 # progress message.
404 if result and result.commit_upto is None:
405 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
406 self.commit_count)
407
408 name += target
Simon Glass4433aa92014-09-05 19:00:07 -0600409 Print(line + name, newline=False)
Simon Glass78e418e2014-08-09 15:33:03 -0600410 length = 14 + len(name)
Simon Glassc05694f2013-04-03 11:07:16 +0000411 self.ClearLine(length)
412
413 def _GetOutputDir(self, commit_upto):
414 """Get the name of the output directory for a commit number
415
416 The output directory is typically .../<branch>/<commit>.
417
418 Args:
419 commit_upto: Commit number to use (0..self.count-1)
420 """
Simon Glasse87bde12014-12-01 17:33:55 -0700421 commit_dir = None
Simon Glassd326ad72014-08-09 15:32:59 -0600422 if self.commits:
423 commit = self.commits[commit_upto]
424 subject = commit.subject.translate(trans_valid_chars)
425 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
426 self.commit_count, commit.hash, subject[:20]))
Simon Glasse87bde12014-12-01 17:33:55 -0700427 elif not self.no_subdirs:
Simon Glassd326ad72014-08-09 15:32:59 -0600428 commit_dir = 'current'
Simon Glasse87bde12014-12-01 17:33:55 -0700429 if not commit_dir:
430 return self.base_dir
431 return os.path.join(self.base_dir, commit_dir)
Simon Glassc05694f2013-04-03 11:07:16 +0000432
433 def GetBuildDir(self, commit_upto, target):
434 """Get the name of the build directory for a commit number
435
436 The build directory is typically .../<branch>/<commit>/<target>.
437
438 Args:
439 commit_upto: Commit number to use (0..self.count-1)
440 target: Target name
441 """
442 output_dir = self._GetOutputDir(commit_upto)
443 return os.path.join(output_dir, target)
444
445 def GetDoneFile(self, commit_upto, target):
446 """Get the name of the done file for a commit number
447
448 Args:
449 commit_upto: Commit number to use (0..self.count-1)
450 target: Target name
451 """
452 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
453
454 def GetSizesFile(self, commit_upto, target):
455 """Get the name of the sizes file for a commit number
456
457 Args:
458 commit_upto: Commit number to use (0..self.count-1)
459 target: Target name
460 """
461 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
462
463 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
464 """Get the name of the funcsizes file for a commit number and ELF file
465
466 Args:
467 commit_upto: Commit number to use (0..self.count-1)
468 target: Target name
469 elf_fname: Filename of elf image
470 """
471 return os.path.join(self.GetBuildDir(commit_upto, target),
472 '%s.sizes' % elf_fname.replace('/', '-'))
473
474 def GetObjdumpFile(self, commit_upto, target, elf_fname):
475 """Get the name of the objdump file for a commit number and ELF file
476
477 Args:
478 commit_upto: Commit number to use (0..self.count-1)
479 target: Target name
480 elf_fname: Filename of elf image
481 """
482 return os.path.join(self.GetBuildDir(commit_upto, target),
483 '%s.objdump' % elf_fname.replace('/', '-'))
484
485 def GetErrFile(self, commit_upto, target):
486 """Get the name of the err file for a commit number
487
488 Args:
489 commit_upto: Commit number to use (0..self.count-1)
490 target: Target name
491 """
492 output_dir = self.GetBuildDir(commit_upto, target)
493 return os.path.join(output_dir, 'err')
494
495 def FilterErrors(self, lines):
496 """Filter out errors in which we have no interest
497
498 We should probably use map().
499
500 Args:
501 lines: List of error lines, each a string
502 Returns:
503 New list with only interesting lines included
504 """
505 out_lines = []
506 for line in lines:
507 if not self.re_make_err.search(line):
508 out_lines.append(line)
509 return out_lines
510
511 def ReadFuncSizes(self, fname, fd):
512 """Read function sizes from the output of 'nm'
513
514 Args:
515 fd: File containing data to read
516 fname: Filename we are reading from (just for errors)
517
518 Returns:
519 Dictionary containing size of each function in bytes, indexed by
520 function name.
521 """
522 sym = {}
523 for line in fd.readlines():
524 try:
525 size, type, name = line[:-1].split()
526 except:
Simon Glass4433aa92014-09-05 19:00:07 -0600527 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassc05694f2013-04-03 11:07:16 +0000528 continue
529 if type in 'tTdDbB':
530 # function names begin with '.' on 64-bit powerpc
531 if '.' in name[1:]:
532 name = 'static.' + name.split('.')[0]
533 sym[name] = sym.get(name, 0) + int(size, 16)
534 return sym
535
Simon Glassdb17fb82015-02-05 22:06:15 -0700536 def _ProcessConfig(self, fname):
537 """Read in a .config, autoconf.mk or autoconf.h file
538
539 This function handles all config file types. It ignores comments and
540 any #defines which don't start with CONFIG_.
541
542 Args:
543 fname: Filename to read
544
545 Returns:
546 Dictionary:
547 key: Config name (e.g. CONFIG_DM)
548 value: Config value (e.g. 1)
549 """
550 config = {}
551 if os.path.exists(fname):
552 with open(fname) as fd:
553 for line in fd:
554 line = line.strip()
555 if line.startswith('#define'):
556 values = line[8:].split(' ', 1)
557 if len(values) > 1:
558 key, value = values
559 else:
560 key = values[0]
561 value = ''
562 if not key.startswith('CONFIG_'):
563 continue
564 elif not line or line[0] in ['#', '*', '/']:
565 continue
566 else:
567 key, value = line.split('=', 1)
568 config[key] = value
569 return config
570
571 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
572 read_config):
Simon Glassc05694f2013-04-03 11:07:16 +0000573 """Work out the outcome of a build.
574
575 Args:
576 commit_upto: Commit number to check (0..n-1)
577 target: Target board to check
578 read_func_sizes: True to read function size information
Simon Glassdb17fb82015-02-05 22:06:15 -0700579 read_config: True to read .config and autoconf.h files
Simon Glassc05694f2013-04-03 11:07:16 +0000580
581 Returns:
582 Outcome object
583 """
584 done_file = self.GetDoneFile(commit_upto, target)
585 sizes_file = self.GetSizesFile(commit_upto, target)
586 sizes = {}
587 func_sizes = {}
Simon Glassdb17fb82015-02-05 22:06:15 -0700588 config = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000589 if os.path.exists(done_file):
590 with open(done_file, 'r') as fd:
591 return_code = int(fd.readline())
592 err_lines = []
593 err_file = self.GetErrFile(commit_upto, target)
594 if os.path.exists(err_file):
595 with open(err_file, 'r') as fd:
596 err_lines = self.FilterErrors(fd.readlines())
597
598 # Decide whether the build was ok, failed or created warnings
599 if return_code:
600 rc = OUTCOME_ERROR
601 elif len(err_lines):
602 rc = OUTCOME_WARNING
603 else:
604 rc = OUTCOME_OK
605
606 # Convert size information to our simple format
607 if os.path.exists(sizes_file):
608 with open(sizes_file, 'r') as fd:
609 for line in fd.readlines():
610 values = line.split()
611 rodata = 0
612 if len(values) > 6:
613 rodata = int(values[6], 16)
614 size_dict = {
615 'all' : int(values[0]) + int(values[1]) +
616 int(values[2]),
617 'text' : int(values[0]) - rodata,
618 'data' : int(values[1]),
619 'bss' : int(values[2]),
620 'rodata' : rodata,
621 }
622 sizes[values[5]] = size_dict
623
624 if read_func_sizes:
625 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
626 for fname in glob.glob(pattern):
627 with open(fname, 'r') as fd:
628 dict_name = os.path.basename(fname).replace('.sizes',
629 '')
630 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
631
Simon Glassdb17fb82015-02-05 22:06:15 -0700632 if read_config:
633 output_dir = self.GetBuildDir(commit_upto, target)
634 for name in CONFIG_FILENAMES:
635 fname = os.path.join(output_dir, name)
636 config[name] = self._ProcessConfig(fname)
637
638 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config)
Simon Glassc05694f2013-04-03 11:07:16 +0000639
Simon Glassdb17fb82015-02-05 22:06:15 -0700640 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {})
Simon Glassc05694f2013-04-03 11:07:16 +0000641
Simon Glassdb17fb82015-02-05 22:06:15 -0700642 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
643 read_config):
Simon Glassc05694f2013-04-03 11:07:16 +0000644 """Calculate a summary of the results of building a commit.
645
646 Args:
647 board_selected: Dict containing boards to summarise
648 commit_upto: Commit number to summarize (0..self.count-1)
649 read_func_sizes: True to read function size information
Simon Glassdb17fb82015-02-05 22:06:15 -0700650 read_config: True to read .config and autoconf.h files
Simon Glassc05694f2013-04-03 11:07:16 +0000651
652 Returns:
653 Tuple:
654 Dict containing boards which passed building this commit.
655 keyed by board.target
Simon Glass03749d42014-08-28 09:43:44 -0600656 List containing a summary of error lines
Simon Glass3394c9f2014-08-28 09:43:43 -0600657 Dict keyed by error line, containing a list of the Board
658 objects with that error
Simon Glass03749d42014-08-28 09:43:44 -0600659 List containing a summary of warning lines
660 Dict keyed by error line, containing a list of the Board
661 objects with that warning
Simon Glassdb17fb82015-02-05 22:06:15 -0700662 Dictionary keyed by filename - e.g. '.config'. Each
663 value is itself a dictionary:
664 key: config name
665 value: config value
Simon Glassc05694f2013-04-03 11:07:16 +0000666 """
Simon Glass03749d42014-08-28 09:43:44 -0600667 def AddLine(lines_summary, lines_boards, line, board):
668 line = line.rstrip()
669 if line in lines_boards:
670 lines_boards[line].append(board)
671 else:
672 lines_boards[line] = [board]
673 lines_summary.append(line)
674
Simon Glassc05694f2013-04-03 11:07:16 +0000675 board_dict = {}
676 err_lines_summary = []
Simon Glass3394c9f2014-08-28 09:43:43 -0600677 err_lines_boards = {}
Simon Glass03749d42014-08-28 09:43:44 -0600678 warn_lines_summary = []
679 warn_lines_boards = {}
Simon Glassdb17fb82015-02-05 22:06:15 -0700680 config = {}
681 for fname in CONFIG_FILENAMES:
682 config[fname] = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000683
684 for board in boards_selected.itervalues():
685 outcome = self.GetBuildOutcome(commit_upto, board.target,
Simon Glassdb17fb82015-02-05 22:06:15 -0700686 read_func_sizes, read_config)
Simon Glassc05694f2013-04-03 11:07:16 +0000687 board_dict[board.target] = outcome
Simon Glass03749d42014-08-28 09:43:44 -0600688 last_func = None
689 last_was_warning = False
690 for line in outcome.err_lines:
691 if line:
692 if (self._re_function.match(line) or
693 self._re_files.match(line)):
694 last_func = line
Simon Glass3394c9f2014-08-28 09:43:43 -0600695 else:
Simon Glass03749d42014-08-28 09:43:44 -0600696 is_warning = self._re_warning.match(line)
697 is_note = self._re_note.match(line)
698 if is_warning or (last_was_warning and is_note):
699 if last_func:
700 AddLine(warn_lines_summary, warn_lines_boards,
701 last_func, board)
702 AddLine(warn_lines_summary, warn_lines_boards,
703 line, board)
704 else:
705 if last_func:
706 AddLine(err_lines_summary, err_lines_boards,
707 last_func, board)
708 AddLine(err_lines_summary, err_lines_boards,
709 line, board)
710 last_was_warning = is_warning
711 last_func = None
Simon Glassdb17fb82015-02-05 22:06:15 -0700712 for fname in CONFIG_FILENAMES:
713 config[fname] = {}
714 if outcome.config:
715 for key, value in outcome.config[fname].iteritems():
716 config[fname][key] = value
717
Simon Glass03749d42014-08-28 09:43:44 -0600718 return (board_dict, err_lines_summary, err_lines_boards,
Simon Glassdb17fb82015-02-05 22:06:15 -0700719 warn_lines_summary, warn_lines_boards, config)
Simon Glassc05694f2013-04-03 11:07:16 +0000720
721 def AddOutcome(self, board_dict, arch_list, changes, char, color):
722 """Add an output to our list of outcomes for each architecture
723
724 This simple function adds failing boards (changes) to the
725 relevant architecture string, so we can print the results out
726 sorted by architecture.
727
728 Args:
729 board_dict: Dict containing all boards
730 arch_list: Dict keyed by arch name. Value is a string containing
731 a list of board names which failed for that arch.
732 changes: List of boards to add to arch_list
733 color: terminal.Colour object
734 """
735 done_arch = {}
736 for target in changes:
737 if target in board_dict:
738 arch = board_dict[target].arch
739 else:
740 arch = 'unknown'
741 str = self.col.Color(color, ' ' + target)
742 if not arch in done_arch:
Simon Glass26f72372015-02-05 22:06:11 -0700743 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassc05694f2013-04-03 11:07:16 +0000744 done_arch[arch] = True
745 if not arch in arch_list:
746 arch_list[arch] = str
747 else:
748 arch_list[arch] += str
749
750
751 def ColourNum(self, num):
752 color = self.col.RED if num > 0 else self.col.GREEN
753 if num == 0:
754 return '0'
755 return self.col.Color(color, str(num))
756
757 def ResetResultSummary(self, board_selected):
758 """Reset the results summary ready for use.
759
760 Set up the base board list to be all those selected, and set the
761 error lines to empty.
762
763 Following this, calls to PrintResultSummary() will use this
764 information to work out what has changed.
765
766 Args:
767 board_selected: Dict containing boards to summarise, keyed by
768 board.target
769 """
770 self._base_board_dict = {}
771 for board in board_selected:
Simon Glassdb17fb82015-02-05 22:06:15 -0700772 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {})
Simon Glassc05694f2013-04-03 11:07:16 +0000773 self._base_err_lines = []
Simon Glass03749d42014-08-28 09:43:44 -0600774 self._base_warn_lines = []
775 self._base_err_line_boards = {}
776 self._base_warn_line_boards = {}
Simon Glassdb17fb82015-02-05 22:06:15 -0700777 self._base_config = {}
778 for fname in CONFIG_FILENAMES:
779 self._base_config[fname] = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000780
781 def PrintFuncSizeDetail(self, fname, old, new):
782 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
783 delta, common = [], {}
784
785 for a in old:
786 if a in new:
787 common[a] = 1
788
789 for name in old:
790 if name not in common:
791 remove += 1
792 down += old[name]
793 delta.append([-old[name], name])
794
795 for name in new:
796 if name not in common:
797 add += 1
798 up += new[name]
799 delta.append([new[name], name])
800
801 for name in common:
802 diff = new.get(name, 0) - old.get(name, 0)
803 if diff > 0:
804 grow, up = grow + 1, up + diff
805 elif diff < 0:
806 shrink, down = shrink + 1, down - diff
807 delta.append([diff, name])
808
809 delta.sort()
810 delta.reverse()
811
812 args = [add, -remove, grow, -shrink, up, -down, up - down]
813 if max(args) == 0:
814 return
815 args = [self.ColourNum(x) for x in args]
816 indent = ' ' * 15
Simon Glass4433aa92014-09-05 19:00:07 -0600817 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
818 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
819 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
820 'delta'))
Simon Glassc05694f2013-04-03 11:07:16 +0000821 for diff, name in delta:
822 if diff:
823 color = self.col.RED if diff > 0 else self.col.GREEN
824 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
825 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4433aa92014-09-05 19:00:07 -0600826 Print(msg, colour=color)
Simon Glassc05694f2013-04-03 11:07:16 +0000827
828
829 def PrintSizeDetail(self, target_list, show_bloat):
830 """Show details size information for each board
831
832 Args:
833 target_list: List of targets, each a dict containing:
834 'target': Target name
835 'total_diff': Total difference in bytes across all areas
836 <part_name>: Difference for that part
837 show_bloat: Show detail for each function
838 """
839 targets_by_diff = sorted(target_list, reverse=True,
840 key=lambda x: x['_total_diff'])
841 for result in targets_by_diff:
842 printed_target = False
843 for name in sorted(result):
844 diff = result[name]
845 if name.startswith('_'):
846 continue
847 if diff != 0:
848 color = self.col.RED if diff > 0 else self.col.GREEN
849 msg = ' %s %+d' % (name, diff)
850 if not printed_target:
Simon Glass4433aa92014-09-05 19:00:07 -0600851 Print('%10s %-15s:' % ('', result['_target']),
852 newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000853 printed_target = True
Simon Glass4433aa92014-09-05 19:00:07 -0600854 Print(msg, colour=color, newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000855 if printed_target:
Simon Glass4433aa92014-09-05 19:00:07 -0600856 Print()
Simon Glassc05694f2013-04-03 11:07:16 +0000857 if show_bloat:
858 target = result['_target']
859 outcome = result['_outcome']
860 base_outcome = self._base_board_dict[target]
861 for fname in outcome.func_sizes:
862 self.PrintFuncSizeDetail(fname,
863 base_outcome.func_sizes[fname],
864 outcome.func_sizes[fname])
865
866
867 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
868 show_bloat):
869 """Print a summary of image sizes broken down by section.
870
871 The summary takes the form of one line per architecture. The
872 line contains deltas for each of the sections (+ means the section
873 got bigger, - means smaller). The nunmbers are the average number
874 of bytes that a board in this section increased by.
875
876 For example:
877 powerpc: (622 boards) text -0.0
878 arm: (285 boards) text -0.0
879 nds32: (3 boards) text -8.0
880
881 Args:
882 board_selected: Dict containing boards to summarise, keyed by
883 board.target
884 board_dict: Dict containing boards for which we built this
885 commit, keyed by board.target. The value is an Outcome object.
886 show_detail: Show detail for each board
887 show_bloat: Show detail for each function
888 """
889 arch_list = {}
890 arch_count = {}
891
892 # Calculate changes in size for different image parts
893 # The previous sizes are in Board.sizes, for each board
894 for target in board_dict:
895 if target not in board_selected:
896 continue
897 base_sizes = self._base_board_dict[target].sizes
898 outcome = board_dict[target]
899 sizes = outcome.sizes
900
901 # Loop through the list of images, creating a dict of size
902 # changes for each image/part. We end up with something like
903 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
904 # which means that U-Boot data increased by 5 bytes and SPL
905 # text decreased by 4.
906 err = {'_target' : target}
907 for image in sizes:
908 if image in base_sizes:
909 base_image = base_sizes[image]
910 # Loop through the text, data, bss parts
911 for part in sorted(sizes[image]):
912 diff = sizes[image][part] - base_image[part]
913 col = None
914 if diff:
915 if image == 'u-boot':
916 name = part
917 else:
918 name = image + ':' + part
919 err[name] = diff
920 arch = board_selected[target].arch
921 if not arch in arch_count:
922 arch_count[arch] = 1
923 else:
924 arch_count[arch] += 1
925 if not sizes:
926 pass # Only add to our list when we have some stats
927 elif not arch in arch_list:
928 arch_list[arch] = [err]
929 else:
930 arch_list[arch].append(err)
931
932 # We now have a list of image size changes sorted by arch
933 # Print out a summary of these
934 for arch, target_list in arch_list.iteritems():
935 # Get total difference for each type
936 totals = {}
937 for result in target_list:
938 total = 0
939 for name, diff in result.iteritems():
940 if name.startswith('_'):
941 continue
942 total += diff
943 if name in totals:
944 totals[name] += diff
945 else:
946 totals[name] = diff
947 result['_total_diff'] = total
948 result['_outcome'] = board_dict[result['_target']]
949
950 count = len(target_list)
951 printed_arch = False
952 for name in sorted(totals):
953 diff = totals[name]
954 if diff:
955 # Display the average difference in this name for this
956 # architecture
957 avg_diff = float(diff) / count
958 color = self.col.RED if avg_diff > 0 else self.col.GREEN
959 msg = ' %s %+1.1f' % (name, avg_diff)
960 if not printed_arch:
Simon Glass4433aa92014-09-05 19:00:07 -0600961 Print('%10s: (for %d/%d boards)' % (arch, count,
962 arch_count[arch]), newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000963 printed_arch = True
Simon Glass4433aa92014-09-05 19:00:07 -0600964 Print(msg, colour=color, newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +0000965
966 if printed_arch:
Simon Glass4433aa92014-09-05 19:00:07 -0600967 Print()
Simon Glassc05694f2013-04-03 11:07:16 +0000968 if show_detail:
969 self.PrintSizeDetail(target_list, show_bloat)
970
971
972 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glass03749d42014-08-28 09:43:44 -0600973 err_line_boards, warn_lines, warn_line_boards,
Simon Glassdb17fb82015-02-05 22:06:15 -0700974 config, show_sizes, show_detail, show_bloat,
975 show_config):
Simon Glassc05694f2013-04-03 11:07:16 +0000976 """Compare results with the base results and display delta.
977
978 Only boards mentioned in board_selected will be considered. This
979 function is intended to be called repeatedly with the results of
980 each commit. It therefore shows a 'diff' between what it saw in
981 the last call and what it sees now.
982
983 Args:
984 board_selected: Dict containing boards to summarise, keyed by
985 board.target
986 board_dict: Dict containing boards for which we built this
987 commit, keyed by board.target. The value is an Outcome object.
988 err_lines: A list of errors for this commit, or [] if there is
989 none, or we don't want to print errors
Simon Glass3394c9f2014-08-28 09:43:43 -0600990 err_line_boards: Dict keyed by error line, containing a list of
991 the Board objects with that error
Simon Glass03749d42014-08-28 09:43:44 -0600992 warn_lines: A list of warnings for this commit, or [] if there is
993 none, or we don't want to print errors
994 warn_line_boards: Dict keyed by warning line, containing a list of
995 the Board objects with that warning
Simon Glassdb17fb82015-02-05 22:06:15 -0700996 config: Dictionary keyed by filename - e.g. '.config'. Each
997 value is itself a dictionary:
998 key: config name
999 value: config value
Simon Glassc05694f2013-04-03 11:07:16 +00001000 show_sizes: Show image size deltas
1001 show_detail: Show detail for each board
1002 show_bloat: Show detail for each function
Simon Glassdb17fb82015-02-05 22:06:15 -07001003 show_config: Show config changes
Simon Glassc05694f2013-04-03 11:07:16 +00001004 """
Simon Glass03749d42014-08-28 09:43:44 -06001005 def _BoardList(line, line_boards):
Simon Glass3394c9f2014-08-28 09:43:43 -06001006 """Helper function to get a line of boards containing a line
1007
1008 Args:
1009 line: Error line to search for
1010 Return:
1011 String containing a list of boards with that error line, or
1012 '' if the user has not requested such a list
1013 """
1014 if self._list_error_boards:
1015 names = []
Simon Glass03749d42014-08-28 09:43:44 -06001016 for board in line_boards[line]:
Simon Glass85620bb2014-10-16 01:05:55 -06001017 if not board.target in names:
1018 names.append(board.target)
Simon Glass3394c9f2014-08-28 09:43:43 -06001019 names_str = '(%s) ' % ','.join(names)
1020 else:
1021 names_str = ''
1022 return names_str
1023
Simon Glass03749d42014-08-28 09:43:44 -06001024 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1025 char):
1026 better_lines = []
1027 worse_lines = []
1028 for line in lines:
1029 if line not in base_lines:
1030 worse_lines.append(char + '+' +
1031 _BoardList(line, line_boards) + line)
1032 for line in base_lines:
1033 if line not in lines:
1034 better_lines.append(char + '-' +
1035 _BoardList(line, base_line_boards) + line)
1036 return better_lines, worse_lines
1037
Simon Glassdb17fb82015-02-05 22:06:15 -07001038 def _CalcConfig(delta, name, config):
1039 """Calculate configuration changes
1040
1041 Args:
1042 delta: Type of the delta, e.g. '+'
1043 name: name of the file which changed (e.g. .config)
1044 config: configuration change dictionary
1045 key: config name
1046 value: config value
1047 Returns:
1048 String containing the configuration changes which can be
1049 printed
1050 """
1051 out = ''
1052 for key in sorted(config.keys()):
1053 out += '%s=%s ' % (key, config[key])
1054 return '%5s %s: %s' % (delta, name, out)
1055
1056 def _ShowConfig(name, config_plus, config_minus, config_change):
1057 """Show changes in configuration
1058
1059 Args:
1060 config_plus: configurations added, dictionary
1061 key: config name
1062 value: config value
1063 config_minus: configurations removed, dictionary
1064 key: config name
1065 value: config value
1066 config_change: configurations changed, dictionary
1067 key: config name
1068 value: config value
1069 """
1070 if config_plus:
1071 Print(_CalcConfig('+', name, config_plus),
1072 colour=self.col.GREEN)
1073 if config_minus:
1074 Print(_CalcConfig('-', name, config_minus),
1075 colour=self.col.RED)
1076 if config_change:
1077 Print(_CalcConfig('+/-', name, config_change),
1078 colour=self.col.YELLOW)
1079
Simon Glassc05694f2013-04-03 11:07:16 +00001080 better = [] # List of boards fixed since last commit
1081 worse = [] # List of new broken boards since last commit
1082 new = [] # List of boards that didn't exist last time
1083 unknown = [] # List of boards that were not built
1084
1085 for target in board_dict:
1086 if target not in board_selected:
1087 continue
1088
1089 # If the board was built last time, add its outcome to a list
1090 if target in self._base_board_dict:
1091 base_outcome = self._base_board_dict[target].rc
1092 outcome = board_dict[target]
1093 if outcome.rc == OUTCOME_UNKNOWN:
1094 unknown.append(target)
1095 elif outcome.rc < base_outcome:
1096 better.append(target)
1097 elif outcome.rc > base_outcome:
1098 worse.append(target)
1099 else:
1100 new.append(target)
1101
1102 # Get a list of errors that have appeared, and disappeared
Simon Glass03749d42014-08-28 09:43:44 -06001103 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1104 self._base_err_line_boards, err_lines, err_line_boards, '')
1105 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1106 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassc05694f2013-04-03 11:07:16 +00001107
1108 # Display results by arch
Simon Glass03749d42014-08-28 09:43:44 -06001109 if (better or worse or unknown or new or worse_err or better_err
1110 or worse_warn or better_warn):
Simon Glassc05694f2013-04-03 11:07:16 +00001111 arch_list = {}
1112 self.AddOutcome(board_selected, arch_list, better, '',
1113 self.col.GREEN)
1114 self.AddOutcome(board_selected, arch_list, worse, '+',
1115 self.col.RED)
1116 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1117 if self._show_unknown:
1118 self.AddOutcome(board_selected, arch_list, unknown, '?',
1119 self.col.MAGENTA)
1120 for arch, target_list in arch_list.iteritems():
Simon Glass4433aa92014-09-05 19:00:07 -06001121 Print('%10s: %s' % (arch, target_list))
Simon Glassbb4dffb2014-08-09 15:33:06 -06001122 self._error_lines += 1
Simon Glassc05694f2013-04-03 11:07:16 +00001123 if better_err:
Simon Glass4433aa92014-09-05 19:00:07 -06001124 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001125 self._error_lines += 1
Simon Glassc05694f2013-04-03 11:07:16 +00001126 if worse_err:
Simon Glass4433aa92014-09-05 19:00:07 -06001127 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001128 self._error_lines += 1
Simon Glass03749d42014-08-28 09:43:44 -06001129 if better_warn:
Simon Glass4433aa92014-09-05 19:00:07 -06001130 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glass03749d42014-08-28 09:43:44 -06001131 self._error_lines += 1
1132 if worse_warn:
Simon Glass4433aa92014-09-05 19:00:07 -06001133 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glass03749d42014-08-28 09:43:44 -06001134 self._error_lines += 1
Simon Glassc05694f2013-04-03 11:07:16 +00001135
1136 if show_sizes:
1137 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1138 show_bloat)
1139
Simon Glassdb17fb82015-02-05 22:06:15 -07001140 if show_config:
1141 all_config_plus = {}
1142 all_config_minus = {}
1143 all_config_change = {}
1144 for name in CONFIG_FILENAMES:
1145 if not config[name]:
1146 continue
1147 config_plus = {}
1148 config_minus = {}
1149 config_change = {}
1150 base = self._base_config[name]
1151 for key, value in config[name].iteritems():
1152 if key not in base:
1153 config_plus[key] = value
1154 all_config_plus[key] = value
1155 for key, value in base.iteritems():
1156 if key not in config[name]:
1157 config_minus[key] = value
1158 all_config_minus[key] = value
1159 for key, value in base.iteritems():
1160 new_value = base[key]
1161 if key in config[name] and value != new_value:
1162 desc = '%s -> %s' % (value, new_value)
1163 config_change[key] = desc
1164 all_config_change[key] = desc
1165 _ShowConfig(name, config_plus, config_minus, config_change)
1166 _ShowConfig('all', all_config_plus, all_config_minus,
1167 all_config_change)
1168
Simon Glassc05694f2013-04-03 11:07:16 +00001169 # Save our updated information for the next call to this function
1170 self._base_board_dict = board_dict
1171 self._base_err_lines = err_lines
Simon Glass03749d42014-08-28 09:43:44 -06001172 self._base_warn_lines = warn_lines
1173 self._base_err_line_boards = err_line_boards
1174 self._base_warn_line_boards = warn_line_boards
Simon Glassdb17fb82015-02-05 22:06:15 -07001175 self._base_config = config
Simon Glassc05694f2013-04-03 11:07:16 +00001176
1177 # Get a list of boards that did not get built, if needed
1178 not_built = []
1179 for board in board_selected:
1180 if not board in board_dict:
1181 not_built.append(board)
1182 if not_built:
Simon Glass4433aa92014-09-05 19:00:07 -06001183 Print("Boards not built (%d): %s" % (len(not_built),
1184 ', '.join(not_built)))
Simon Glassc05694f2013-04-03 11:07:16 +00001185
Simon Glasseb48bbc2014-08-09 15:33:02 -06001186 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glass03749d42014-08-28 09:43:44 -06001187 (board_dict, err_lines, err_line_boards, warn_lines,
Simon Glassdb17fb82015-02-05 22:06:15 -07001188 warn_line_boards, config) = self.GetResultSummary(
Simon Glass3394c9f2014-08-28 09:43:43 -06001189 board_selected, commit_upto,
Simon Glassdb17fb82015-02-05 22:06:15 -07001190 read_func_sizes=self._show_bloat,
1191 read_config=self._show_config)
Simon Glasseb48bbc2014-08-09 15:33:02 -06001192 if commits:
1193 msg = '%02d: %s' % (commit_upto + 1,
1194 commits[commit_upto].subject)
Simon Glass4433aa92014-09-05 19:00:07 -06001195 Print(msg, colour=self.col.BLUE)
Simon Glasseb48bbc2014-08-09 15:33:02 -06001196 self.PrintResultSummary(board_selected, board_dict,
Simon Glass3394c9f2014-08-28 09:43:43 -06001197 err_lines if self._show_errors else [], err_line_boards,
Simon Glass03749d42014-08-28 09:43:44 -06001198 warn_lines if self._show_errors else [], warn_line_boards,
Simon Glassdb17fb82015-02-05 22:06:15 -07001199 config, self._show_sizes, self._show_detail,
1200 self._show_bloat, self._show_config)
Simon Glassc05694f2013-04-03 11:07:16 +00001201
Simon Glasseb48bbc2014-08-09 15:33:02 -06001202 def ShowSummary(self, commits, board_selected):
Simon Glassc05694f2013-04-03 11:07:16 +00001203 """Show a build summary for U-Boot for a given board list.
1204
1205 Reset the result summary, then repeatedly call GetResultSummary on
1206 each commit's results, then display the differences we see.
1207
1208 Args:
1209 commit: Commit objects to summarise
1210 board_selected: Dict containing boards to summarise
Simon Glassc05694f2013-04-03 11:07:16 +00001211 """
Simon Glassd326ad72014-08-09 15:32:59 -06001212 self.commit_count = len(commits) if commits else 1
Simon Glassc05694f2013-04-03 11:07:16 +00001213 self.commits = commits
1214 self.ResetResultSummary(board_selected)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001215 self._error_lines = 0
Simon Glassc05694f2013-04-03 11:07:16 +00001216
1217 for commit_upto in range(0, self.commit_count, self._step):
Simon Glasseb48bbc2014-08-09 15:33:02 -06001218 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001219 if not self._error_lines:
Simon Glass4433aa92014-09-05 19:00:07 -06001220 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassc05694f2013-04-03 11:07:16 +00001221
1222
1223 def SetupBuild(self, board_selected, commits):
1224 """Set up ready to start a build.
1225
1226 Args:
1227 board_selected: Selected boards to build
1228 commits: Selected commits to build
1229 """
1230 # First work out how many commits we will build
Simon Glassd326ad72014-08-09 15:32:59 -06001231 count = (self.commit_count + self._step - 1) / self._step
Simon Glassc05694f2013-04-03 11:07:16 +00001232 self.count = len(board_selected) * count
1233 self.upto = self.warned = self.fail = 0
1234 self._timestamps = collections.deque()
1235
Simon Glassc05694f2013-04-03 11:07:16 +00001236 def GetThreadDir(self, thread_num):
1237 """Get the directory path to the working dir for a thread.
1238
1239 Args:
1240 thread_num: Number of thread to check.
1241 """
1242 return os.path.join(self._working_dir, '%02d' % thread_num)
1243
Simon Glassd326ad72014-08-09 15:32:59 -06001244 def _PrepareThread(self, thread_num, setup_git):
Simon Glassc05694f2013-04-03 11:07:16 +00001245 """Prepare the working directory for a thread.
1246
1247 This clones or fetches the repo into the thread's work directory.
1248
1249 Args:
1250 thread_num: Thread number (0, 1, ...)
Simon Glassd326ad72014-08-09 15:32:59 -06001251 setup_git: True to set up a git repo clone
Simon Glassc05694f2013-04-03 11:07:16 +00001252 """
1253 thread_dir = self.GetThreadDir(thread_num)
Simon Glass4a1e88b2014-08-09 15:33:00 -06001254 builderthread.Mkdir(thread_dir)
Simon Glassc05694f2013-04-03 11:07:16 +00001255 git_dir = os.path.join(thread_dir, '.git')
1256
1257 # Clone the repo if it doesn't already exist
1258 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1259 # we have a private index but uses the origin repo's contents?
Simon Glassd326ad72014-08-09 15:32:59 -06001260 if setup_git and self.git_dir:
Simon Glassc05694f2013-04-03 11:07:16 +00001261 src_dir = os.path.abspath(self.git_dir)
1262 if os.path.exists(git_dir):
1263 gitutil.Fetch(git_dir, thread_dir)
1264 else:
Simon Glass4433aa92014-09-05 19:00:07 -06001265 Print('Cloning repo for thread %d' % thread_num)
Simon Glassc05694f2013-04-03 11:07:16 +00001266 gitutil.Clone(src_dir, thread_dir)
1267
Simon Glassd326ad72014-08-09 15:32:59 -06001268 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassc05694f2013-04-03 11:07:16 +00001269 """Prepare the working directory for use.
1270
1271 Set up the git repo for each thread.
1272
1273 Args:
1274 max_threads: Maximum number of threads we expect to need.
Simon Glassd326ad72014-08-09 15:32:59 -06001275 setup_git: True to set up a git repo clone
Simon Glassc05694f2013-04-03 11:07:16 +00001276 """
Simon Glass4a1e88b2014-08-09 15:33:00 -06001277 builderthread.Mkdir(self._working_dir)
Simon Glassc05694f2013-04-03 11:07:16 +00001278 for thread in range(max_threads):
Simon Glassd326ad72014-08-09 15:32:59 -06001279 self._PrepareThread(thread, setup_git)
Simon Glassc05694f2013-04-03 11:07:16 +00001280
1281 def _PrepareOutputSpace(self):
1282 """Get the output directories ready to receive files.
1283
1284 We delete any output directories which look like ones we need to
1285 create. Having left over directories is confusing when the user wants
1286 to check the output manually.
1287 """
Simon Glassb9a9b7c2014-12-01 17:33:53 -07001288 if not self.commits:
1289 return
Simon Glassc05694f2013-04-03 11:07:16 +00001290 dir_list = []
1291 for commit_upto in range(self.commit_count):
1292 dir_list.append(self._GetOutputDir(commit_upto))
1293
1294 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1295 if dirname not in dir_list:
1296 shutil.rmtree(dirname)
1297
Simon Glass78e418e2014-08-09 15:33:03 -06001298 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassc05694f2013-04-03 11:07:16 +00001299 """Build all commits for a list of boards
1300
1301 Args:
1302 commits: List of commits to be build, each a Commit object
1303 boards_selected: Dict of selected boards, key is target name,
1304 value is Board object
Simon Glassc05694f2013-04-03 11:07:16 +00001305 keep_outputs: True to save build output files
Simon Glass78e418e2014-08-09 15:33:03 -06001306 verbose: Display build results as they are completed
Simon Glassc2f91072014-08-28 09:43:39 -06001307 Returns:
1308 Tuple containing:
1309 - number of boards that failed to build
1310 - number of boards that issued warnings
Simon Glassc05694f2013-04-03 11:07:16 +00001311 """
Simon Glassd326ad72014-08-09 15:32:59 -06001312 self.commit_count = len(commits) if commits else 1
Simon Glassc05694f2013-04-03 11:07:16 +00001313 self.commits = commits
Simon Glass78e418e2014-08-09 15:33:03 -06001314 self._verbose = verbose
Simon Glassc05694f2013-04-03 11:07:16 +00001315
1316 self.ResetResultSummary(board_selected)
Thierry Reding336d5bd2014-08-19 10:22:39 +02001317 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassd326ad72014-08-09 15:32:59 -06001318 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1319 commits is not None)
Simon Glassc05694f2013-04-03 11:07:16 +00001320 self._PrepareOutputSpace()
1321 self.SetupBuild(board_selected, commits)
1322 self.ProcessResult(None)
1323
1324 # Create jobs to build all commits for each board
1325 for brd in board_selected.itervalues():
Simon Glass4a1e88b2014-08-09 15:33:00 -06001326 job = builderthread.BuilderJob()
Simon Glassc05694f2013-04-03 11:07:16 +00001327 job.board = brd
1328 job.commits = commits
1329 job.keep_outputs = keep_outputs
1330 job.step = self._step
1331 self.queue.put(job)
1332
1333 # Wait until all jobs are started
1334 self.queue.join()
1335
1336 # Wait until we have processed all output
1337 self.out_queue.join()
Simon Glass4433aa92014-09-05 19:00:07 -06001338 Print()
Simon Glassc05694f2013-04-03 11:07:16 +00001339 self.ClearLine(0)
Simon Glassc2f91072014-08-28 09:43:39 -06001340 return (self.fail, self.warned)