blob: c4384f53e8dc783f81d6f55b14339c2372839ce1 [file] [log] [blame]
Tom Rini10e47792018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassc05694f2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5#
Simon Glassc05694f2013-04-03 11:07:16 +00006
7import collections
Simon Glassc05694f2013-04-03 11:07:16 +00008from datetime import datetime, timedelta
9import glob
10import os
11import re
Simon Glassc78ed662019-10-31 07:42:53 -060012import queue
Simon Glassc05694f2013-04-03 11:07:16 +000013import shutil
Simon Glass205ac042016-09-18 16:48:37 -060014import signal
Simon Glassc05694f2013-04-03 11:07:16 +000015import string
16import sys
Simon Glassd26e1442016-09-18 16:48:35 -060017import threading
Simon Glassc05694f2013-04-03 11:07:16 +000018import time
19
Simon Glassf0d9c102020-04-17 18:09:02 -060020from buildman import builderthread
21from buildman import toolchain
Simon Glassa997ea52020-04-17 18:09:04 -060022from patman import gitutil
Simon Glass131444f2023-02-23 18:18:04 -070023from u_boot_pylib import command
24from u_boot_pylib import terminal
25from u_boot_pylib.terminal import tprint
Simon Glassc05694f2013-04-03 11:07:16 +000026
Simon Glass146b6022021-10-19 21:43:24 -060027# This indicates an new int or hex Kconfig property with no default
28# It hangs the build since the 'conf' tool cannot proceed without valid input.
29#
30# We get a repeat sequence of something like this:
31# >>
32# Break things (BREAK_ME) [] (NEW)
33# Error in reading or end of file.
34# <<
35# which indicates that BREAK_ME has an empty default
36RE_NO_DEFAULT = re.compile(b'\((\w+)\) \[] \(NEW\)')
37
Simon Glassdac73712023-10-23 00:52:43 -070038# Symbol types which appear in the bloat feature (-B). Others are silently
39# dropped when reading in the 'nm' output
40NM_SYMBOL_TYPES = 'tTdDbBr'
41
Simon Glassc05694f2013-04-03 11:07:16 +000042"""
43Theory of Operation
44
45Please see README for user documentation, and you should be familiar with
46that before trying to make sense of this.
47
48Buildman works by keeping the machine as busy as possible, building different
49commits for different boards on multiple CPUs at once.
50
51The source repo (self.git_dir) contains all the commits to be built. Each
52thread works on a single board at a time. It checks out the first commit,
53configures it for that board, then builds it. Then it checks out the next
54commit and builds it (typically without re-configuring). When it runs out
55of commits, it gets another job from the builder and starts again with that
56board.
57
58Clearly the builder threads could work either way - they could check out a
59commit and then built it for all boards. Using separate directories for each
60commit/board pair they could leave their build product around afterwards
61also.
62
63The intent behind building a single board for multiple commits, is to make
64use of incremental builds. Since each commit is built incrementally from
65the previous one, builds are faster. Reconfiguring for a different board
66removes all intermediate object files.
67
68Many threads can be working at once, but each has its own working directory.
69When a thread finishes a build, it puts the output files into a result
70directory.
71
72The base directory used by buildman is normally '../<branch>', i.e.
73a directory higher than the source repository and named after the branch
74being built.
75
76Within the base directory, we have one subdirectory for each commit. Within
77that is one subdirectory for each board. Within that is the build output for
78that commit/board combination.
79
80Buildman also create working directories for each thread, in a .bm-work/
81subdirectory in the base dir.
82
83As an example, say we are building branch 'us-net' for boards 'sandbox' and
84'seaboard', and say that us-net has two commits. We will have directories
85like this:
86
87us-net/ base directory
Ovidiu Panaitee8e9cb2020-05-15 09:30:12 +030088 01_g4ed4ebc_net--Add-tftp-speed-/
Simon Glassc05694f2013-04-03 11:07:16 +000089 sandbox/
90 u-boot.bin
91 seaboard/
92 u-boot.bin
Ovidiu Panaitee8e9cb2020-05-15 09:30:12 +030093 02_g4ed4ebc_net--Check-tftp-comp/
Simon Glassc05694f2013-04-03 11:07:16 +000094 sandbox/
95 u-boot.bin
96 seaboard/
97 u-boot.bin
98 .bm-work/
99 00/ working directory for thread 0 (contains source checkout)
100 build/ build output
101 01/ working directory for thread 1
102 build/ build output
103 ...
104u-boot/ source directory
105 .git/ repository
106"""
107
Simon Glassde0fefc2020-04-09 15:08:36 -0600108"""Holds information about a particular error line we are outputing
109
110 char: Character representation: '+': error, '-': fixed error, 'w+': warning,
111 'w-' = fixed warning
112 boards: List of Board objects which have line in the error/warning output
113 errline: The text of the error line
114"""
Simon Glass5df45222022-07-11 19:04:00 -0600115ErrLine = collections.namedtuple('ErrLine', 'char,brds,errline')
Simon Glassde0fefc2020-04-09 15:08:36 -0600116
Simon Glassc05694f2013-04-03 11:07:16 +0000117# Possible build outcomes
Simon Glassc78ed662019-10-31 07:42:53 -0600118OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
Simon Glassc05694f2013-04-03 11:07:16 +0000119
Simon Glassd214bef2017-04-12 18:23:26 -0600120# Translate a commit subject into a valid filename (and handle unicode)
Simon Glassc78ed662019-10-31 07:42:53 -0600121trans_valid_chars = str.maketrans('/: ', '---')
Simon Glassc05694f2013-04-03 11:07:16 +0000122
Simon Glasscde5c302016-11-13 14:25:53 -0700123BASE_CONFIG_FILENAMES = [
124 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
125]
126
127EXTRA_CONFIG_FILENAMES = [
Simon Glassdb17fb82015-02-05 22:06:15 -0700128 '.config', '.config-spl', '.config-tpl',
129 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
130 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glassdb17fb82015-02-05 22:06:15 -0700131]
132
Simon Glasscad8abf2015-08-25 21:52:14 -0600133class Config:
134 """Holds information about configuration settings for a board."""
Simon Glasscde5c302016-11-13 14:25:53 -0700135 def __init__(self, config_filename, target):
Simon Glasscad8abf2015-08-25 21:52:14 -0600136 self.target = target
137 self.config = {}
Simon Glasscde5c302016-11-13 14:25:53 -0700138 for fname in config_filename:
Simon Glasscad8abf2015-08-25 21:52:14 -0600139 self.config[fname] = {}
140
Simon Glassbc74d942023-07-19 17:49:06 -0600141 def add(self, fname, key, value):
Simon Glasscad8abf2015-08-25 21:52:14 -0600142 self.config[fname][key] = value
143
144 def __hash__(self):
145 val = 0
146 for fname in self.config:
Simon Glassc78ed662019-10-31 07:42:53 -0600147 for key, value in self.config[fname].items():
148 print(key, value)
Simon Glasscad8abf2015-08-25 21:52:14 -0600149 val = val ^ hash(key) & hash(value)
150 return val
Simon Glassc05694f2013-04-03 11:07:16 +0000151
Alex Kiernan4059e302018-05-31 04:48:34 +0000152class Environment:
153 """Holds information about environment variables for a board."""
154 def __init__(self, target):
155 self.target = target
156 self.environment = {}
157
Simon Glassbc74d942023-07-19 17:49:06 -0600158 def add(self, key, value):
Alex Kiernan4059e302018-05-31 04:48:34 +0000159 self.environment[key] = value
160
Simon Glassc05694f2013-04-03 11:07:16 +0000161class Builder:
162 """Class for building U-Boot for a particular commit.
163
164 Public members: (many should ->private)
Simon Glassc05694f2013-04-03 11:07:16 +0000165 already_done: Number of builds already completed
166 base_dir: Base directory to use for builder
167 checkout: True to check out source, False to skip that step.
168 This is used for testing.
169 col: terminal.Color() object
Simon Glassf56cc292023-07-19 17:49:03 -0600170 count: Total number of commits to build, which is the number of commits
171 multiplied by the number of boards
Simon Glassc05694f2013-04-03 11:07:16 +0000172 do_make: Method to call to invoke Make
173 fail: Number of builds that failed due to error
174 force_build: Force building even if a build already exists
175 force_config_on_failure: If a commit fails for a board, disable
176 incremental building for the next commit we build for that
177 board, so that we will see all warnings/errors again.
Simon Glass7041c392014-07-13 12:22:31 -0600178 force_build_failures: If a previously-built build (i.e. built on
179 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassc05694f2013-04-03 11:07:16 +0000180 git_dir: Git directory containing source repository
Simon Glassc05694f2013-04-03 11:07:16 +0000181 num_jobs: Number of jobs to run at once (passed to make as -j)
182 num_threads: Number of builder threads to run
183 out_queue: Queue of results to process
184 re_make_err: Compiled regular expression for ignore_lines
185 queue: Queue of jobs to run
186 threads: List of active threads
187 toolchains: Toolchains object to use for building
188 upto: Current commit number we are building (0.count-1)
189 warned: Number of builds that produced at least one warning
Simon Glassf3018b7a2014-07-14 17:51:02 -0600190 force_reconfig: Reconfigure U-Boot on each comiit. This disables
191 incremental building, where buildman reconfigures on the first
192 commit for a baord, and then just does an incremental build for
193 the following commits. In fact buildman will reconfigure and
194 retry for any failing commits, so generally the only effect of
195 this option is to slow things down.
Simon Glass38df2e22014-07-14 17:51:03 -0600196 in_tree: Build U-Boot in-tree instead of specifying an output
197 directory separate from the source code. This option is really
198 only useful for testing in-tree builds.
Simon Glassb6eb8cf2020-03-18 09:42:42 -0600199 work_in_output: Use the output directory as the work directory and
200 don't write to a separate output directory.
Simon Glass9bf9a722021-04-11 16:27:27 +1200201 thread_exceptions: List of exceptions raised by thread jobs
Simon Glassf6bfcca2023-02-21 12:40:28 -0700202 no_lto (bool): True to set the NO_LTO flag when building
Simon Glass828d70d2023-02-21 12:40:29 -0700203 reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds
Simon Glassc05694f2013-04-03 11:07:16 +0000204
205 Private members:
206 _base_board_dict: Last-summarised Dict of boards
207 _base_err_lines: Last-summarised list of errors
Simon Glass03749d42014-08-28 09:43:44 -0600208 _base_warn_lines: Last-summarised list of warnings
Simon Glassc05694f2013-04-03 11:07:16 +0000209 _build_period_us: Time taken for a single build (float object).
210 _complete_delay: Expected delay until completion (timedelta)
211 _next_delay_update: Next time we plan to display a progress update
212 (datatime)
213 _show_unknown: Show unknown boards (those not built) in summary
Simon Glass726ae812020-04-09 15:08:47 -0600214 _start_time: Start time for the build
Simon Glassc05694f2013-04-03 11:07:16 +0000215 _timestamps: List of timestamps for the completion of the last
216 last _timestamp_count builds. Each is a datetime object.
217 _timestamp_count: Number of timestamps to keep in our list.
218 _working_dir: Base working directory containing all threads
Simon Glassc635d892021-01-30 22:17:46 -0700219 _single_builder: BuilderThread object for the singer builder, if
220 threading is not being used
Simon Glass146b6022021-10-19 21:43:24 -0600221 _terminated: Thread was terminated due to an error
222 _restarting_config: True if 'Restart config' is detected in output
Simon Glass6c435622022-07-11 19:03:56 -0600223 _ide: Produce output suitable for an Integrated Development Environment,
224 i.e. dont emit progress information and put errors/warnings on stderr
Simon Glassc05694f2013-04-03 11:07:16 +0000225 """
226 class Outcome:
227 """Records a build outcome for a single make invocation
228
229 Public Members:
230 rc: Outcome value (OUTCOME_...)
231 err_lines: List of error lines or [] if none
232 sizes: Dictionary of image size information, keyed by filename
233 - Each value is itself a dictionary containing
234 values for 'text', 'data' and 'bss', being the integer
235 size in bytes of each section.
236 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
237 value is itself a dictionary:
238 key: function name
239 value: Size of function in bytes
Simon Glassdb17fb82015-02-05 22:06:15 -0700240 config: Dictionary keyed by filename - e.g. '.config'. Each
241 value is itself a dictionary:
242 key: config name
243 value: config value
Alex Kiernan4059e302018-05-31 04:48:34 +0000244 environment: Dictionary keyed by environment variable, Each
245 value is the value of environment variable.
Simon Glassc05694f2013-04-03 11:07:16 +0000246 """
Alex Kiernan4059e302018-05-31 04:48:34 +0000247 def __init__(self, rc, err_lines, sizes, func_sizes, config,
248 environment):
Simon Glassc05694f2013-04-03 11:07:16 +0000249 self.rc = rc
250 self.err_lines = err_lines
251 self.sizes = sizes
252 self.func_sizes = func_sizes
Simon Glassdb17fb82015-02-05 22:06:15 -0700253 self.config = config
Alex Kiernan4059e302018-05-31 04:48:34 +0000254 self.environment = environment
Simon Glassc05694f2013-04-03 11:07:16 +0000255
256 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glasse87bde12014-12-01 17:33:55 -0700257 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warren97c96902016-04-11 10:48:44 -0600258 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glass222825b2024-06-23 11:55:13 -0600259 mrproper=False, fallback_mrproper=False,
260 per_board_out_dir=False, config_only=False,
261 squash_config_y=False, warnings_as_errors=False,
262 work_in_output=False, test_thread_exceptions=False,
263 adjust_cfg=None, allow_missing=False, no_lto=False,
264 reproducible_builds=False, force_build=False,
265 force_build_failures=False, force_reconfig=False,
266 in_tree=False, force_config_on_failure=False, make_func=None):
Simon Glassc05694f2013-04-03 11:07:16 +0000267 """Create a new Builder object
268
269 Args:
270 toolchains: Toolchains object to use for building
271 base_dir: Base directory to use for builder
272 git_dir: Git directory containing source repository
273 num_threads: Number of builder threads to run
274 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900275 gnu_make: the command name of GNU Make.
Simon Glassc05694f2013-04-03 11:07:16 +0000276 checkout: True to check out source, False to skip that step.
277 This is used for testing.
278 show_unknown: Show unknown boards (those not built) in summary
279 step: 1 to process every commit, n to process every nth commit
Simon Glassd48a46c2014-12-01 17:34:00 -0700280 no_subdirs: Don't create subdirectories when building current
281 source for a single board
282 full_path: Return the full path in CROSS_COMPILE and don't set
283 PATH
Simon Glass655b6102014-12-01 17:34:07 -0700284 verbose_build: Run build with V=1 and don't use 'make -s'
Simon Glass6029af12020-04-09 15:08:51 -0600285 mrproper: Always run 'make mrproper' when configuring
Simon Glass222825b2024-06-23 11:55:13 -0600286 fallback_mrproper: Run 'make mrproper' and retry on build failure
Stephen Warren97c96902016-04-11 10:48:44 -0600287 per_board_out_dir: Build in a separate persistent directory per
288 board rather than a thread-specific directory
Simon Glass739e8512016-11-13 14:25:51 -0700289 config_only: Only configure each build, don't build it
Simon Glasscde5c302016-11-13 14:25:53 -0700290 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck20e2ea92018-01-26 16:31:05 +0100291 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassb6eb8cf2020-03-18 09:42:42 -0600292 work_in_output: Use the output directory as the work directory and
293 don't write to a separate output directory.
Simon Glass9bf9a722021-04-11 16:27:27 +1200294 test_thread_exceptions: Uses for tests only, True to make the
295 threads raise an exception instead of reporting their result.
296 This simulates a failure in the code somewhere
Simon Glasse5650a82022-01-22 05:07:33 -0700297 adjust_cfg_list (list of str): List of changes to make to .config
298 file before building. Each is one of (where C is the config
299 option with or without the CONFIG_ prefix)
300
301 C to enable C
302 ~C to disable C
303 C=val to set the value of C (val must have quotes if C is
304 a string Kconfig
Tom Rini93ebd462022-11-09 19:14:53 -0700305 allow_missing: Run build with BINMAN_ALLOW_MISSING=1
Simon Glassf6bfcca2023-02-21 12:40:28 -0700306 no_lto (bool): True to set the NO_LTO flag when building
Simon Glasscf91d312023-07-19 17:48:52 -0600307 force_build (bool): Rebuild even commits that are already built
308 force_build_failures (bool): Rebuild commits that have not been
309 built, or failed to build
310 force_reconfig (bool): Reconfigure on each commit
311 in_tree (bool): Bulid in tree instead of out-of-tree
312 force_config_on_failure (bool): Reconfigure the build before
313 retrying a failed build
314 make_func (function): Function to call to run 'make'
Simon Glassc05694f2013-04-03 11:07:16 +0000315 """
316 self.toolchains = toolchains
317 self.base_dir = base_dir
Simon Glassb6eb8cf2020-03-18 09:42:42 -0600318 if work_in_output:
319 self._working_dir = base_dir
320 else:
321 self._working_dir = os.path.join(base_dir, '.bm-work')
Simon Glassc05694f2013-04-03 11:07:16 +0000322 self.threads = []
Simon Glassbc74d942023-07-19 17:49:06 -0600323 self.do_make = make_func or self.make
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900324 self.gnu_make = gnu_make
Simon Glassc05694f2013-04-03 11:07:16 +0000325 self.checkout = checkout
326 self.num_threads = num_threads
327 self.num_jobs = num_jobs
328 self.already_done = 0
329 self.force_build = False
330 self.git_dir = git_dir
331 self._show_unknown = show_unknown
332 self._timestamp_count = 10
333 self._build_period_us = None
334 self._complete_delay = None
335 self._next_delay_update = datetime.now()
Simon Glass7190a172023-09-07 10:00:19 -0600336 self._start_time = None
Simon Glassc05694f2013-04-03 11:07:16 +0000337 self._step = step
Simon Glassbb4dffb2014-08-09 15:33:06 -0600338 self._error_lines = 0
Simon Glasse87bde12014-12-01 17:33:55 -0700339 self.no_subdirs = no_subdirs
Simon Glassd48a46c2014-12-01 17:34:00 -0700340 self.full_path = full_path
Simon Glass655b6102014-12-01 17:34:07 -0700341 self.verbose_build = verbose_build
Simon Glass739e8512016-11-13 14:25:51 -0700342 self.config_only = config_only
Simon Glasscde5c302016-11-13 14:25:53 -0700343 self.squash_config_y = squash_config_y
344 self.config_filenames = BASE_CONFIG_FILENAMES
Simon Glassb6eb8cf2020-03-18 09:42:42 -0600345 self.work_in_output = work_in_output
Simon Glasse5650a82022-01-22 05:07:33 -0700346 self.adjust_cfg = adjust_cfg
Tom Rini93ebd462022-11-09 19:14:53 -0700347 self.allow_missing = allow_missing
Simon Glass6c435622022-07-11 19:03:56 -0600348 self._ide = False
Simon Glassf6bfcca2023-02-21 12:40:28 -0700349 self.no_lto = no_lto
Simon Glass828d70d2023-02-21 12:40:29 -0700350 self.reproducible_builds = reproducible_builds
Simon Glasscf91d312023-07-19 17:48:52 -0600351 self.force_build = force_build
352 self.force_build_failures = force_build_failures
353 self.force_reconfig = force_reconfig
354 self.in_tree = in_tree
355 self.force_config_on_failure = force_config_on_failure
Simon Glass222825b2024-06-23 11:55:13 -0600356 self.fallback_mrproper = fallback_mrproper
Simon Glasse5650a82022-01-22 05:07:33 -0700357
Simon Glasscde5c302016-11-13 14:25:53 -0700358 if not self.squash_config_y:
359 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glass146b6022021-10-19 21:43:24 -0600360 self._terminated = False
361 self._restarting_config = False
Simon Glassc05694f2013-04-03 11:07:16 +0000362
Daniel Schwierzeck20e2ea92018-01-26 16:31:05 +0100363 self.warnings_as_errors = warnings_as_errors
Simon Glassc05694f2013-04-03 11:07:16 +0000364 self.col = terminal.Color()
365
Simon Glass03749d42014-08-28 09:43:44 -0600366 self._re_function = re.compile('(.*): In function.*')
367 self._re_files = re.compile('In file included from.*')
368 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
Simon Glass0db94432018-11-06 16:02:11 -0700369 self._re_dtb_warning = re.compile('(.*): Warning .*')
Simon Glass03749d42014-08-28 09:43:44 -0600370 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
Simon Glassf4ebfba2020-04-09 15:08:53 -0600371 self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
372 re.MULTILINE | re.DOTALL)
Simon Glass03749d42014-08-28 09:43:44 -0600373
Simon Glass9bf9a722021-04-11 16:27:27 +1200374 self.thread_exceptions = []
375 self.test_thread_exceptions = test_thread_exceptions
Simon Glassc635d892021-01-30 22:17:46 -0700376 if self.num_threads:
377 self._single_builder = None
378 self.queue = queue.Queue()
379 self.out_queue = queue.Queue()
380 for i in range(self.num_threads):
Simon Glass9bf9a722021-04-11 16:27:27 +1200381 t = builderthread.BuilderThread(
382 self, i, mrproper, per_board_out_dir,
383 test_exception=test_thread_exceptions)
Simon Glassc635d892021-01-30 22:17:46 -0700384 t.setDaemon(True)
385 t.start()
386 self.threads.append(t)
387
388 t = builderthread.ResultThread(self)
Simon Glassc05694f2013-04-03 11:07:16 +0000389 t.setDaemon(True)
390 t.start()
391 self.threads.append(t)
Simon Glassc635d892021-01-30 22:17:46 -0700392 else:
393 self._single_builder = builderthread.BuilderThread(
394 self, -1, mrproper, per_board_out_dir)
Simon Glassc05694f2013-04-03 11:07:16 +0000395
396 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
397 self.re_make_err = re.compile('|'.join(ignore_lines))
398
Simon Glass205ac042016-09-18 16:48:37 -0600399 # Handle existing graceful with SIGINT / Ctrl-C
400 signal.signal(signal.SIGINT, self.signal_handler)
401
Simon Glassc05694f2013-04-03 11:07:16 +0000402 def __del__(self):
403 """Get rid of all threads created by the builder"""
404 for t in self.threads:
405 del t
406
Simon Glass205ac042016-09-18 16:48:37 -0600407 def signal_handler(self, signal, frame):
408 sys.exit(1)
409
Simon Glassbc74d942023-07-19 17:49:06 -0600410 def set_display_options(self, show_errors=False, show_sizes=False,
Simon Glass3394c9f2014-08-28 09:43:43 -0600411 show_detail=False, show_bloat=False,
Alex Kiernan4059e302018-05-31 04:48:34 +0000412 list_error_boards=False, show_config=False,
Simon Glassf4ebfba2020-04-09 15:08:53 -0600413 show_environment=False, filter_dtb_warnings=False,
Simon Glass6c435622022-07-11 19:03:56 -0600414 filter_migration_warnings=False, ide=False):
Simon Glasseb48bbc2014-08-09 15:33:02 -0600415 """Setup display options for the builder.
416
Simon Glass9ea93812020-04-09 15:08:52 -0600417 Args:
418 show_errors: True to show summarised error/warning info
419 show_sizes: Show size deltas
420 show_detail: Show size delta detail for each board if show_sizes
421 show_bloat: Show detail for each function
422 list_error_boards: Show the boards which caused each error/warning
423 show_config: Show config deltas
424 show_environment: Show environment deltas
425 filter_dtb_warnings: Filter out any warnings from the device-tree
426 compiler
Simon Glassf4ebfba2020-04-09 15:08:53 -0600427 filter_migration_warnings: Filter out any warnings about migrating
428 a board to driver model
Simon Glass6c435622022-07-11 19:03:56 -0600429 ide: Create output that can be parsed by an IDE. There is no '+' prefix on
430 error lines and output on stderr stays on stderr.
Simon Glasseb48bbc2014-08-09 15:33:02 -0600431 """
432 self._show_errors = show_errors
433 self._show_sizes = show_sizes
434 self._show_detail = show_detail
435 self._show_bloat = show_bloat
Simon Glass3394c9f2014-08-28 09:43:43 -0600436 self._list_error_boards = list_error_boards
Simon Glassdb17fb82015-02-05 22:06:15 -0700437 self._show_config = show_config
Alex Kiernan4059e302018-05-31 04:48:34 +0000438 self._show_environment = show_environment
Simon Glass9ea93812020-04-09 15:08:52 -0600439 self._filter_dtb_warnings = filter_dtb_warnings
Simon Glassf4ebfba2020-04-09 15:08:53 -0600440 self._filter_migration_warnings = filter_migration_warnings
Simon Glass6c435622022-07-11 19:03:56 -0600441 self._ide = ide
Simon Glasseb48bbc2014-08-09 15:33:02 -0600442
Simon Glassbc74d942023-07-19 17:49:06 -0600443 def _add_timestamp(self):
Simon Glassc05694f2013-04-03 11:07:16 +0000444 """Add a new timestamp to the list and record the build period.
445
446 The build period is the length of time taken to perform a single
447 build (one board, one commit).
448 """
449 now = datetime.now()
450 self._timestamps.append(now)
451 count = len(self._timestamps)
452 delta = self._timestamps[-1] - self._timestamps[0]
453 seconds = delta.total_seconds()
454
455 # If we have enough data, estimate build period (time taken for a
456 # single build) and therefore completion time.
457 if count > 1 and self._next_delay_update < now:
458 self._next_delay_update = now + timedelta(seconds=2)
459 if seconds > 0:
460 self._build_period = float(seconds) / count
461 todo = self.count - self.upto
462 self._complete_delay = timedelta(microseconds=
463 self._build_period * todo * 1000000)
464 # Round it
465 self._complete_delay -= timedelta(
466 microseconds=self._complete_delay.microseconds)
467
468 if seconds > 60:
469 self._timestamps.popleft()
470 count -= 1
471
Simon Glassbc74d942023-07-19 17:49:06 -0600472 def select_commit(self, commit, checkout=True):
Simon Glassc05694f2013-04-03 11:07:16 +0000473 """Checkout the selected commit for this build
474 """
475 self.commit = commit
476 if checkout and self.checkout:
Simon Glass761648b2022-01-29 14:14:11 -0700477 gitutil.checkout(commit.hash)
Simon Glassc05694f2013-04-03 11:07:16 +0000478
Simon Glassbc74d942023-07-19 17:49:06 -0600479 def make(self, commit, brd, stage, cwd, *args, **kwargs):
Simon Glassc05694f2013-04-03 11:07:16 +0000480 """Run make
481
482 Args:
483 commit: Commit object that is being built
484 brd: Board object that is being built
Simon Glassd6c1ec82023-10-26 14:31:10 -0400485 stage: Stage that we are at (mrproper, config, oldconfig, build)
Simon Glassc05694f2013-04-03 11:07:16 +0000486 cwd: Directory where make should be run
487 args: Arguments to pass to make
Simon Glass840be732022-01-29 14:14:05 -0700488 kwargs: Arguments to pass to command.run_pipe()
Simon Glassc05694f2013-04-03 11:07:16 +0000489 """
Simon Glass146b6022021-10-19 21:43:24 -0600490
491 def check_output(stream, data):
492 if b'Restart config' in data:
493 self._restarting_config = True
494
495 # If we see 'Restart config' following by multiple errors
496 if self._restarting_config:
497 m = RE_NO_DEFAULT.findall(data)
498
499 # Number of occurences of each Kconfig item
500 multiple = [m.count(val) for val in set(m)]
501
502 # If any of them occur more than once, we have a loop
503 if [val for val in multiple if val > 1]:
504 self._terminated = True
505 return True
506 return False
507
508 self._restarting_config = False
509 self._terminated = False
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900510 cmd = [self.gnu_make] + list(args)
Simon Glass840be732022-01-29 14:14:05 -0700511 result = command.run_pipe([cmd], capture=True, capture_stderr=True,
Simon Glass146b6022021-10-19 21:43:24 -0600512 cwd=cwd, raise_on_error=False, infile='/dev/null',
513 output_func=check_output, **kwargs)
514
515 if self._terminated:
516 # Try to be helpful
517 result.stderr += '(** did you define an int/hex Kconfig with no default? **)'
518
Simon Glass413f91a2015-02-05 22:06:12 -0700519 if self.verbose_build:
520 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
521 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassc05694f2013-04-03 11:07:16 +0000522 return result
523
Simon Glassbc74d942023-07-19 17:49:06 -0600524 def process_result(self, result):
Simon Glassc05694f2013-04-03 11:07:16 +0000525 """Process the result of a build, showing progress information
526
527 Args:
Simon Glass78e418e2014-08-09 15:33:03 -0600528 result: A CommandResult object, which indicates the result for
529 a single build
Simon Glassc05694f2013-04-03 11:07:16 +0000530 """
531 col = terminal.Color()
532 if result:
533 target = result.brd.target
534
Simon Glassc05694f2013-04-03 11:07:16 +0000535 self.upto += 1
536 if result.return_code != 0:
537 self.fail += 1
538 elif result.stderr:
539 self.warned += 1
540 if result.already_done:
541 self.already_done += 1
Simon Glass78e418e2014-08-09 15:33:03 -0600542 if self._verbose:
Simon Glass02811582022-01-29 14:14:18 -0700543 terminal.print_clear()
Simon Glass78e418e2014-08-09 15:33:03 -0600544 boards_selected = {target : result.brd}
Simon Glassbc74d942023-07-19 17:49:06 -0600545 self.reset_result_summary(boards_selected)
546 self.produce_result_summary(result.commit_upto, self.commits,
Simon Glass78e418e2014-08-09 15:33:03 -0600547 boards_selected)
Simon Glassc05694f2013-04-03 11:07:16 +0000548 else:
549 target = '(starting)'
550
551 # Display separate counts for ok, warned and fail
552 ok = self.upto - self.warned - self.fail
Simon Glassf45d3742022-01-29 14:14:17 -0700553 line = '\r' + self.col.build(self.col.GREEN, '%5d' % ok)
554 line += self.col.build(self.col.YELLOW, '%5d' % self.warned)
555 line += self.col.build(self.col.RED, '%5d' % self.fail)
Simon Glassc05694f2013-04-03 11:07:16 +0000556
Simon Glass69c3a8a2020-04-09 15:08:45 -0600557 line += ' /%-5d ' % self.count
558 remaining = self.count - self.upto
559 if remaining:
Simon Glassf45d3742022-01-29 14:14:17 -0700560 line += self.col.build(self.col.MAGENTA, ' -%-5d ' % remaining)
Simon Glass69c3a8a2020-04-09 15:08:45 -0600561 else:
562 line += ' ' * 8
Simon Glassc05694f2013-04-03 11:07:16 +0000563
564 # Add our current completion time estimate
Simon Glassbc74d942023-07-19 17:49:06 -0600565 self._add_timestamp()
Simon Glassc05694f2013-04-03 11:07:16 +0000566 if self._complete_delay:
Simon Glass69c3a8a2020-04-09 15:08:45 -0600567 line += '%s : ' % self._complete_delay
Simon Glassc05694f2013-04-03 11:07:16 +0000568
Simon Glass69c3a8a2020-04-09 15:08:45 -0600569 line += target
Simon Glass6c435622022-07-11 19:03:56 -0600570 if not self._ide:
571 terminal.print_clear()
572 tprint(line, newline=False, limit_to_line=True)
Simon Glassc05694f2013-04-03 11:07:16 +0000573
Simon Glass4cb54682023-07-19 17:49:10 -0600574 def get_output_dir(self, commit_upto):
Simon Glassc05694f2013-04-03 11:07:16 +0000575 """Get the name of the output directory for a commit number
576
577 The output directory is typically .../<branch>/<commit>.
578
579 Args:
580 commit_upto: Commit number to use (0..self.count-1)
581 """
Simon Glasse3c85ab2020-04-17 17:51:34 -0600582 if self.work_in_output:
583 return self._working_dir
584
Simon Glasse87bde12014-12-01 17:33:55 -0700585 commit_dir = None
Simon Glassd326ad72014-08-09 15:32:59 -0600586 if self.commits:
587 commit = self.commits[commit_upto]
588 subject = commit.subject.translate(trans_valid_chars)
Simon Glassbc74d942023-07-19 17:49:06 -0600589 # See _get_output_space_removals() which parses this name
Ovidiu Panaitee8e9cb2020-05-15 09:30:12 +0300590 commit_dir = ('%02d_g%s_%s' % (commit_upto + 1,
591 commit.hash, subject[:20]))
Simon Glasse87bde12014-12-01 17:33:55 -0700592 elif not self.no_subdirs:
Simon Glassd326ad72014-08-09 15:32:59 -0600593 commit_dir = 'current'
Simon Glasse87bde12014-12-01 17:33:55 -0700594 if not commit_dir:
595 return self.base_dir
596 return os.path.join(self.base_dir, commit_dir)
Simon Glassc05694f2013-04-03 11:07:16 +0000597
Simon Glassbc74d942023-07-19 17:49:06 -0600598 def get_build_dir(self, commit_upto, target):
Simon Glassc05694f2013-04-03 11:07:16 +0000599 """Get the name of the build directory for a commit number
600
601 The build directory is typically .../<branch>/<commit>/<target>.
602
603 Args:
604 commit_upto: Commit number to use (0..self.count-1)
605 target: Target name
606 """
Simon Glass4cb54682023-07-19 17:49:10 -0600607 output_dir = self.get_output_dir(commit_upto)
Simon Glasse3c85ab2020-04-17 17:51:34 -0600608 if self.work_in_output:
609 return output_dir
Simon Glassc05694f2013-04-03 11:07:16 +0000610 return os.path.join(output_dir, target)
611
Simon Glassbc74d942023-07-19 17:49:06 -0600612 def get_done_file(self, commit_upto, target):
Simon Glassc05694f2013-04-03 11:07:16 +0000613 """Get the name of the done file for a commit number
614
615 Args:
616 commit_upto: Commit number to use (0..self.count-1)
617 target: Target name
618 """
Simon Glassbc74d942023-07-19 17:49:06 -0600619 return os.path.join(self.get_build_dir(commit_upto, target), 'done')
Simon Glassc05694f2013-04-03 11:07:16 +0000620
Simon Glassbc74d942023-07-19 17:49:06 -0600621 def get_sizes_file(self, commit_upto, target):
Simon Glassc05694f2013-04-03 11:07:16 +0000622 """Get the name of the sizes file for a commit number
623
624 Args:
625 commit_upto: Commit number to use (0..self.count-1)
626 target: Target name
627 """
Simon Glassbc74d942023-07-19 17:49:06 -0600628 return os.path.join(self.get_build_dir(commit_upto, target), 'sizes')
Simon Glassc05694f2013-04-03 11:07:16 +0000629
Simon Glassbc74d942023-07-19 17:49:06 -0600630 def get_func_sizes_file(self, commit_upto, target, elf_fname):
Simon Glassc05694f2013-04-03 11:07:16 +0000631 """Get the name of the funcsizes file for a commit number and ELF file
632
633 Args:
634 commit_upto: Commit number to use (0..self.count-1)
635 target: Target name
636 elf_fname: Filename of elf image
637 """
Simon Glassbc74d942023-07-19 17:49:06 -0600638 return os.path.join(self.get_build_dir(commit_upto, target),
Simon Glassc05694f2013-04-03 11:07:16 +0000639 '%s.sizes' % elf_fname.replace('/', '-'))
640
Simon Glassbc74d942023-07-19 17:49:06 -0600641 def get_objdump_file(self, commit_upto, target, elf_fname):
Simon Glassc05694f2013-04-03 11:07:16 +0000642 """Get the name of the objdump file for a commit number and ELF file
643
644 Args:
645 commit_upto: Commit number to use (0..self.count-1)
646 target: Target name
647 elf_fname: Filename of elf image
648 """
Simon Glassbc74d942023-07-19 17:49:06 -0600649 return os.path.join(self.get_build_dir(commit_upto, target),
Simon Glassc05694f2013-04-03 11:07:16 +0000650 '%s.objdump' % elf_fname.replace('/', '-'))
651
Simon Glassbc74d942023-07-19 17:49:06 -0600652 def get_err_file(self, commit_upto, target):
Simon Glassc05694f2013-04-03 11:07:16 +0000653 """Get the name of the err file for a commit number
654
655 Args:
656 commit_upto: Commit number to use (0..self.count-1)
657 target: Target name
658 """
Simon Glassbc74d942023-07-19 17:49:06 -0600659 output_dir = self.get_build_dir(commit_upto, target)
Simon Glassc05694f2013-04-03 11:07:16 +0000660 return os.path.join(output_dir, 'err')
661
Simon Glassbc74d942023-07-19 17:49:06 -0600662 def filter_errors(self, lines):
Simon Glassc05694f2013-04-03 11:07:16 +0000663 """Filter out errors in which we have no interest
664
665 We should probably use map().
666
667 Args:
668 lines: List of error lines, each a string
669 Returns:
670 New list with only interesting lines included
671 """
672 out_lines = []
Simon Glassf4ebfba2020-04-09 15:08:53 -0600673 if self._filter_migration_warnings:
674 text = '\n'.join(lines)
675 text = self._re_migration_warning.sub('', text)
676 lines = text.splitlines()
Simon Glassc05694f2013-04-03 11:07:16 +0000677 for line in lines:
Simon Glass9ea93812020-04-09 15:08:52 -0600678 if self.re_make_err.search(line):
679 continue
680 if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
681 continue
682 out_lines.append(line)
Simon Glassc05694f2013-04-03 11:07:16 +0000683 return out_lines
684
Simon Glassbc74d942023-07-19 17:49:06 -0600685 def read_func_sizes(self, fname, fd):
Simon Glassc05694f2013-04-03 11:07:16 +0000686 """Read function sizes from the output of 'nm'
687
688 Args:
689 fd: File containing data to read
690 fname: Filename we are reading from (just for errors)
691
692 Returns:
693 Dictionary containing size of each function in bytes, indexed by
694 function name.
695 """
696 sym = {}
697 for line in fd.readlines():
Simon Glass86a2afe2022-07-11 19:04:11 -0600698 line = line.strip()
699 parts = line.split()
700 if line and len(parts) == 3:
701 size, type, name = line.split()
Simon Glassdac73712023-10-23 00:52:43 -0700702 if type in NM_SYMBOL_TYPES:
Simon Glass86a2afe2022-07-11 19:04:11 -0600703 # function names begin with '.' on 64-bit powerpc
704 if '.' in name[1:]:
705 name = 'static.' + name.split('.')[0]
706 sym[name] = sym.get(name, 0) + int(size, 16)
Simon Glassc05694f2013-04-03 11:07:16 +0000707 return sym
708
Simon Glassbc74d942023-07-19 17:49:06 -0600709 def _process_config(self, fname):
Simon Glassdb17fb82015-02-05 22:06:15 -0700710 """Read in a .config, autoconf.mk or autoconf.h file
711
712 This function handles all config file types. It ignores comments and
713 any #defines which don't start with CONFIG_.
714
715 Args:
716 fname: Filename to read
717
718 Returns:
719 Dictionary:
720 key: Config name (e.g. CONFIG_DM)
721 value: Config value (e.g. 1)
722 """
723 config = {}
724 if os.path.exists(fname):
725 with open(fname) as fd:
726 for line in fd:
727 line = line.strip()
728 if line.startswith('#define'):
729 values = line[8:].split(' ', 1)
730 if len(values) > 1:
731 key, value = values
732 else:
733 key = values[0]
Simon Glasscde5c302016-11-13 14:25:53 -0700734 value = '1' if self.squash_config_y else ''
Simon Glassdb17fb82015-02-05 22:06:15 -0700735 if not key.startswith('CONFIG_'):
736 continue
737 elif not line or line[0] in ['#', '*', '/']:
738 continue
739 else:
740 key, value = line.split('=', 1)
Simon Glasscde5c302016-11-13 14:25:53 -0700741 if self.squash_config_y and value == 'y':
742 value = '1'
Simon Glassdb17fb82015-02-05 22:06:15 -0700743 config[key] = value
744 return config
745
Simon Glassbc74d942023-07-19 17:49:06 -0600746 def _process_environment(self, fname):
Alex Kiernan4059e302018-05-31 04:48:34 +0000747 """Read in a uboot.env file
748
749 This function reads in environment variables from a file.
750
751 Args:
752 fname: Filename to read
753
754 Returns:
755 Dictionary:
756 key: environment variable (e.g. bootlimit)
757 value: value of environment variable (e.g. 1)
758 """
759 environment = {}
760 if os.path.exists(fname):
761 with open(fname) as fd:
762 for line in fd.read().split('\0'):
763 try:
764 key, value = line.split('=', 1)
765 environment[key] = value
766 except ValueError:
767 # ignore lines we can't parse
768 pass
769 return environment
770
Simon Glassbc74d942023-07-19 17:49:06 -0600771 def get_build_outcome(self, commit_upto, target, read_func_sizes,
Alex Kiernan4059e302018-05-31 04:48:34 +0000772 read_config, read_environment):
Simon Glassc05694f2013-04-03 11:07:16 +0000773 """Work out the outcome of a build.
774
775 Args:
776 commit_upto: Commit number to check (0..n-1)
777 target: Target board to check
778 read_func_sizes: True to read function size information
Simon Glassdb17fb82015-02-05 22:06:15 -0700779 read_config: True to read .config and autoconf.h files
Alex Kiernan4059e302018-05-31 04:48:34 +0000780 read_environment: True to read uboot.env files
Simon Glassc05694f2013-04-03 11:07:16 +0000781
782 Returns:
783 Outcome object
784 """
Simon Glassbc74d942023-07-19 17:49:06 -0600785 done_file = self.get_done_file(commit_upto, target)
786 sizes_file = self.get_sizes_file(commit_upto, target)
Simon Glassc05694f2013-04-03 11:07:16 +0000787 sizes = {}
788 func_sizes = {}
Simon Glassdb17fb82015-02-05 22:06:15 -0700789 config = {}
Alex Kiernan4059e302018-05-31 04:48:34 +0000790 environment = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000791 if os.path.exists(done_file):
792 with open(done_file, 'r') as fd:
Simon Glass91c54a42019-04-26 19:02:23 -0600793 try:
794 return_code = int(fd.readline())
795 except ValueError:
796 # The file may be empty due to running out of disk space.
797 # Try a rebuild
798 return_code = 1
Simon Glassc05694f2013-04-03 11:07:16 +0000799 err_lines = []
Simon Glassbc74d942023-07-19 17:49:06 -0600800 err_file = self.get_err_file(commit_upto, target)
Simon Glassc05694f2013-04-03 11:07:16 +0000801 if os.path.exists(err_file):
802 with open(err_file, 'r') as fd:
Simon Glassbc74d942023-07-19 17:49:06 -0600803 err_lines = self.filter_errors(fd.readlines())
Simon Glassc05694f2013-04-03 11:07:16 +0000804
805 # Decide whether the build was ok, failed or created warnings
806 if return_code:
807 rc = OUTCOME_ERROR
808 elif len(err_lines):
809 rc = OUTCOME_WARNING
810 else:
811 rc = OUTCOME_OK
812
813 # Convert size information to our simple format
814 if os.path.exists(sizes_file):
815 with open(sizes_file, 'r') as fd:
816 for line in fd.readlines():
817 values = line.split()
818 rodata = 0
819 if len(values) > 6:
820 rodata = int(values[6], 16)
821 size_dict = {
822 'all' : int(values[0]) + int(values[1]) +
823 int(values[2]),
824 'text' : int(values[0]) - rodata,
825 'data' : int(values[1]),
826 'bss' : int(values[2]),
827 'rodata' : rodata,
828 }
829 sizes[values[5]] = size_dict
830
831 if read_func_sizes:
Simon Glassbc74d942023-07-19 17:49:06 -0600832 pattern = self.get_func_sizes_file(commit_upto, target, '*')
Simon Glassc05694f2013-04-03 11:07:16 +0000833 for fname in glob.glob(pattern):
834 with open(fname, 'r') as fd:
835 dict_name = os.path.basename(fname).replace('.sizes',
836 '')
Simon Glassbc74d942023-07-19 17:49:06 -0600837 func_sizes[dict_name] = self.read_func_sizes(fname, fd)
Simon Glassc05694f2013-04-03 11:07:16 +0000838
Simon Glassdb17fb82015-02-05 22:06:15 -0700839 if read_config:
Simon Glassbc74d942023-07-19 17:49:06 -0600840 output_dir = self.get_build_dir(commit_upto, target)
Simon Glasscde5c302016-11-13 14:25:53 -0700841 for name in self.config_filenames:
Simon Glassdb17fb82015-02-05 22:06:15 -0700842 fname = os.path.join(output_dir, name)
Simon Glassbc74d942023-07-19 17:49:06 -0600843 config[name] = self._process_config(fname)
Simon Glassdb17fb82015-02-05 22:06:15 -0700844
Alex Kiernan4059e302018-05-31 04:48:34 +0000845 if read_environment:
Simon Glassbc74d942023-07-19 17:49:06 -0600846 output_dir = self.get_build_dir(commit_upto, target)
Alex Kiernan4059e302018-05-31 04:48:34 +0000847 fname = os.path.join(output_dir, 'uboot.env')
Simon Glassbc74d942023-07-19 17:49:06 -0600848 environment = self._process_environment(fname)
Alex Kiernan4059e302018-05-31 04:48:34 +0000849
850 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
851 environment)
Simon Glassc05694f2013-04-03 11:07:16 +0000852
Alex Kiernan4059e302018-05-31 04:48:34 +0000853 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
Simon Glassc05694f2013-04-03 11:07:16 +0000854
Simon Glassbc74d942023-07-19 17:49:06 -0600855 def get_result_summary(self, boards_selected, commit_upto, read_func_sizes,
Alex Kiernan4059e302018-05-31 04:48:34 +0000856 read_config, read_environment):
Simon Glassc05694f2013-04-03 11:07:16 +0000857 """Calculate a summary of the results of building a commit.
858
859 Args:
860 board_selected: Dict containing boards to summarise
861 commit_upto: Commit number to summarize (0..self.count-1)
862 read_func_sizes: True to read function size information
Simon Glassdb17fb82015-02-05 22:06:15 -0700863 read_config: True to read .config and autoconf.h files
Alex Kiernan4059e302018-05-31 04:48:34 +0000864 read_environment: True to read uboot.env files
Simon Glassc05694f2013-04-03 11:07:16 +0000865
866 Returns:
867 Tuple:
Simon Glass6c435622022-07-11 19:03:56 -0600868 Dict containing boards which built this commit:
869 key: board.target
870 value: Builder.Outcome object
Simon Glass03749d42014-08-28 09:43:44 -0600871 List containing a summary of error lines
Simon Glass3394c9f2014-08-28 09:43:43 -0600872 Dict keyed by error line, containing a list of the Board
873 objects with that error
Simon Glass03749d42014-08-28 09:43:44 -0600874 List containing a summary of warning lines
875 Dict keyed by error line, containing a list of the Board
876 objects with that warning
Simon Glasscad8abf2015-08-25 21:52:14 -0600877 Dictionary keyed by board.target. Each value is a dictionary:
878 key: filename - e.g. '.config'
Simon Glassdb17fb82015-02-05 22:06:15 -0700879 value is itself a dictionary:
880 key: config name
881 value: config value
Alex Kiernan4059e302018-05-31 04:48:34 +0000882 Dictionary keyed by board.target. Each value is a dictionary:
883 key: environment variable
884 value: value of environment variable
Simon Glassc05694f2013-04-03 11:07:16 +0000885 """
Simon Glassbc74d942023-07-19 17:49:06 -0600886 def add_line(lines_summary, lines_boards, line, board):
Simon Glass03749d42014-08-28 09:43:44 -0600887 line = line.rstrip()
888 if line in lines_boards:
889 lines_boards[line].append(board)
890 else:
891 lines_boards[line] = [board]
892 lines_summary.append(line)
893
Simon Glassc05694f2013-04-03 11:07:16 +0000894 board_dict = {}
895 err_lines_summary = []
Simon Glass3394c9f2014-08-28 09:43:43 -0600896 err_lines_boards = {}
Simon Glass03749d42014-08-28 09:43:44 -0600897 warn_lines_summary = []
898 warn_lines_boards = {}
Simon Glassdb17fb82015-02-05 22:06:15 -0700899 config = {}
Alex Kiernan4059e302018-05-31 04:48:34 +0000900 environment = {}
Simon Glassc05694f2013-04-03 11:07:16 +0000901
Simon Glass8132f982022-07-11 19:03:57 -0600902 for brd in boards_selected.values():
Simon Glassbc74d942023-07-19 17:49:06 -0600903 outcome = self.get_build_outcome(commit_upto, brd.target,
Alex Kiernan4059e302018-05-31 04:48:34 +0000904 read_func_sizes, read_config,
905 read_environment)
Simon Glass8132f982022-07-11 19:03:57 -0600906 board_dict[brd.target] = outcome
Simon Glass03749d42014-08-28 09:43:44 -0600907 last_func = None
908 last_was_warning = False
909 for line in outcome.err_lines:
910 if line:
911 if (self._re_function.match(line) or
912 self._re_files.match(line)):
913 last_func = line
Simon Glass3394c9f2014-08-28 09:43:43 -0600914 else:
Simon Glass0db94432018-11-06 16:02:11 -0700915 is_warning = (self._re_warning.match(line) or
916 self._re_dtb_warning.match(line))
Simon Glass03749d42014-08-28 09:43:44 -0600917 is_note = self._re_note.match(line)
918 if is_warning or (last_was_warning and is_note):
919 if last_func:
Simon Glassbc74d942023-07-19 17:49:06 -0600920 add_line(warn_lines_summary, warn_lines_boards,
Simon Glass8132f982022-07-11 19:03:57 -0600921 last_func, brd)
Simon Glassbc74d942023-07-19 17:49:06 -0600922 add_line(warn_lines_summary, warn_lines_boards,
Simon Glass8132f982022-07-11 19:03:57 -0600923 line, brd)
Simon Glass03749d42014-08-28 09:43:44 -0600924 else:
925 if last_func:
Simon Glassbc74d942023-07-19 17:49:06 -0600926 add_line(err_lines_summary, err_lines_boards,
Simon Glass8132f982022-07-11 19:03:57 -0600927 last_func, brd)
Simon Glassbc74d942023-07-19 17:49:06 -0600928 add_line(err_lines_summary, err_lines_boards,
Simon Glass8132f982022-07-11 19:03:57 -0600929 line, brd)
Simon Glass03749d42014-08-28 09:43:44 -0600930 last_was_warning = is_warning
931 last_func = None
Simon Glass8132f982022-07-11 19:03:57 -0600932 tconfig = Config(self.config_filenames, brd.target)
Simon Glasscde5c302016-11-13 14:25:53 -0700933 for fname in self.config_filenames:
Simon Glassdb17fb82015-02-05 22:06:15 -0700934 if outcome.config:
Simon Glassc78ed662019-10-31 07:42:53 -0600935 for key, value in outcome.config[fname].items():
Simon Glassbc74d942023-07-19 17:49:06 -0600936 tconfig.add(fname, key, value)
Simon Glass8132f982022-07-11 19:03:57 -0600937 config[brd.target] = tconfig
Simon Glassdb17fb82015-02-05 22:06:15 -0700938
Simon Glass8132f982022-07-11 19:03:57 -0600939 tenvironment = Environment(brd.target)
Alex Kiernan4059e302018-05-31 04:48:34 +0000940 if outcome.environment:
Simon Glassc78ed662019-10-31 07:42:53 -0600941 for key, value in outcome.environment.items():
Simon Glassbc74d942023-07-19 17:49:06 -0600942 tenvironment.add(key, value)
Simon Glass8132f982022-07-11 19:03:57 -0600943 environment[brd.target] = tenvironment
Alex Kiernan4059e302018-05-31 04:48:34 +0000944
Simon Glass03749d42014-08-28 09:43:44 -0600945 return (board_dict, err_lines_summary, err_lines_boards,
Alex Kiernan4059e302018-05-31 04:48:34 +0000946 warn_lines_summary, warn_lines_boards, config, environment)
Simon Glassc05694f2013-04-03 11:07:16 +0000947
Simon Glassbc74d942023-07-19 17:49:06 -0600948 def add_outcome(self, board_dict, arch_list, changes, char, color):
Simon Glassc05694f2013-04-03 11:07:16 +0000949 """Add an output to our list of outcomes for each architecture
950
951 This simple function adds failing boards (changes) to the
952 relevant architecture string, so we can print the results out
953 sorted by architecture.
954
955 Args:
956 board_dict: Dict containing all boards
957 arch_list: Dict keyed by arch name. Value is a string containing
958 a list of board names which failed for that arch.
959 changes: List of boards to add to arch_list
960 color: terminal.Colour object
961 """
962 done_arch = {}
963 for target in changes:
964 if target in board_dict:
965 arch = board_dict[target].arch
966 else:
967 arch = 'unknown'
Simon Glassf45d3742022-01-29 14:14:17 -0700968 str = self.col.build(color, ' ' + target)
Simon Glassc05694f2013-04-03 11:07:16 +0000969 if not arch in done_arch:
Simon Glassf45d3742022-01-29 14:14:17 -0700970 str = ' %s %s' % (self.col.build(color, char), str)
Simon Glassc05694f2013-04-03 11:07:16 +0000971 done_arch[arch] = True
972 if not arch in arch_list:
973 arch_list[arch] = str
974 else:
975 arch_list[arch] += str
976
977
Simon Glassbc74d942023-07-19 17:49:06 -0600978 def colour_num(self, num):
Simon Glassc05694f2013-04-03 11:07:16 +0000979 color = self.col.RED if num > 0 else self.col.GREEN
980 if num == 0:
981 return '0'
Simon Glassf45d3742022-01-29 14:14:17 -0700982 return self.col.build(color, str(num))
Simon Glassc05694f2013-04-03 11:07:16 +0000983
Simon Glassbc74d942023-07-19 17:49:06 -0600984 def reset_result_summary(self, board_selected):
Simon Glassc05694f2013-04-03 11:07:16 +0000985 """Reset the results summary ready for use.
986
987 Set up the base board list to be all those selected, and set the
988 error lines to empty.
989
Simon Glassbc74d942023-07-19 17:49:06 -0600990 Following this, calls to print_result_summary() will use this
Simon Glassc05694f2013-04-03 11:07:16 +0000991 information to work out what has changed.
992
993 Args:
994 board_selected: Dict containing boards to summarise, keyed by
995 board.target
996 """
997 self._base_board_dict = {}
Simon Glass8132f982022-07-11 19:03:57 -0600998 for brd in board_selected:
999 self._base_board_dict[brd] = Builder.Outcome(0, [], [], {}, {}, {})
Simon Glassc05694f2013-04-03 11:07:16 +00001000 self._base_err_lines = []
Simon Glass03749d42014-08-28 09:43:44 -06001001 self._base_warn_lines = []
1002 self._base_err_line_boards = {}
1003 self._base_warn_line_boards = {}
Simon Glasscad8abf2015-08-25 21:52:14 -06001004 self._base_config = None
Alex Kiernan4059e302018-05-31 04:48:34 +00001005 self._base_environment = None
Simon Glassc05694f2013-04-03 11:07:16 +00001006
Simon Glassbc74d942023-07-19 17:49:06 -06001007 def print_func_size_detail(self, fname, old, new):
Simon Glassc05694f2013-04-03 11:07:16 +00001008 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
1009 delta, common = [], {}
1010
1011 for a in old:
1012 if a in new:
1013 common[a] = 1
1014
1015 for name in old:
1016 if name not in common:
1017 remove += 1
1018 down += old[name]
1019 delta.append([-old[name], name])
1020
1021 for name in new:
1022 if name not in common:
1023 add += 1
1024 up += new[name]
1025 delta.append([new[name], name])
1026
1027 for name in common:
1028 diff = new.get(name, 0) - old.get(name, 0)
1029 if diff > 0:
1030 grow, up = grow + 1, up + diff
1031 elif diff < 0:
1032 shrink, down = shrink + 1, down - diff
1033 delta.append([diff, name])
1034
1035 delta.sort()
1036 delta.reverse()
1037
1038 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rini0b48cd62017-05-22 13:48:52 -04001039 if max(args) == 0 and min(args) == 0:
Simon Glassc05694f2013-04-03 11:07:16 +00001040 return
Simon Glassbc74d942023-07-19 17:49:06 -06001041 args = [self.colour_num(x) for x in args]
Simon Glassc05694f2013-04-03 11:07:16 +00001042 indent = ' ' * 15
Simon Glass02811582022-01-29 14:14:18 -07001043 tprint('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
Simon Glassf45d3742022-01-29 14:14:17 -07001044 tuple([indent, self.col.build(self.col.YELLOW, fname)] + args))
Simon Glass02811582022-01-29 14:14:18 -07001045 tprint('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
Simon Glass4433aa92014-09-05 19:00:07 -06001046 'delta'))
Simon Glassc05694f2013-04-03 11:07:16 +00001047 for diff, name in delta:
1048 if diff:
1049 color = self.col.RED if diff > 0 else self.col.GREEN
1050 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
1051 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass02811582022-01-29 14:14:18 -07001052 tprint(msg, colour=color)
Simon Glassc05694f2013-04-03 11:07:16 +00001053
1054
Simon Glassbc74d942023-07-19 17:49:06 -06001055 def print_size_detail(self, target_list, show_bloat):
Simon Glassc05694f2013-04-03 11:07:16 +00001056 """Show details size information for each board
1057
1058 Args:
1059 target_list: List of targets, each a dict containing:
1060 'target': Target name
1061 'total_diff': Total difference in bytes across all areas
1062 <part_name>: Difference for that part
1063 show_bloat: Show detail for each function
1064 """
1065 targets_by_diff = sorted(target_list, reverse=True,
1066 key=lambda x: x['_total_diff'])
1067 for result in targets_by_diff:
1068 printed_target = False
1069 for name in sorted(result):
1070 diff = result[name]
1071 if name.startswith('_'):
1072 continue
1073 if diff != 0:
1074 color = self.col.RED if diff > 0 else self.col.GREEN
1075 msg = ' %s %+d' % (name, diff)
1076 if not printed_target:
Simon Glass02811582022-01-29 14:14:18 -07001077 tprint('%10s %-15s:' % ('', result['_target']),
Simon Glass4433aa92014-09-05 19:00:07 -06001078 newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +00001079 printed_target = True
Simon Glass02811582022-01-29 14:14:18 -07001080 tprint(msg, colour=color, newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +00001081 if printed_target:
Simon Glass02811582022-01-29 14:14:18 -07001082 tprint()
Simon Glassc05694f2013-04-03 11:07:16 +00001083 if show_bloat:
1084 target = result['_target']
1085 outcome = result['_outcome']
1086 base_outcome = self._base_board_dict[target]
1087 for fname in outcome.func_sizes:
Simon Glassbc74d942023-07-19 17:49:06 -06001088 self.print_func_size_detail(fname,
Simon Glassc05694f2013-04-03 11:07:16 +00001089 base_outcome.func_sizes[fname],
1090 outcome.func_sizes[fname])
1091
1092
Simon Glassbc74d942023-07-19 17:49:06 -06001093 def print_size_summary(self, board_selected, board_dict, show_detail,
Simon Glassc05694f2013-04-03 11:07:16 +00001094 show_bloat):
1095 """Print a summary of image sizes broken down by section.
1096
1097 The summary takes the form of one line per architecture. The
1098 line contains deltas for each of the sections (+ means the section
Flavio Suligoie5e3c112020-01-29 09:56:05 +01001099 got bigger, - means smaller). The numbers are the average number
Simon Glassc05694f2013-04-03 11:07:16 +00001100 of bytes that a board in this section increased by.
1101
1102 For example:
1103 powerpc: (622 boards) text -0.0
1104 arm: (285 boards) text -0.0
Simon Glassc05694f2013-04-03 11:07:16 +00001105
1106 Args:
1107 board_selected: Dict containing boards to summarise, keyed by
1108 board.target
1109 board_dict: Dict containing boards for which we built this
1110 commit, keyed by board.target. The value is an Outcome object.
Simon Glassb4002462020-03-18 09:42:43 -06001111 show_detail: Show size delta detail for each board
Simon Glassc05694f2013-04-03 11:07:16 +00001112 show_bloat: Show detail for each function
1113 """
1114 arch_list = {}
1115 arch_count = {}
1116
1117 # Calculate changes in size for different image parts
1118 # The previous sizes are in Board.sizes, for each board
1119 for target in board_dict:
1120 if target not in board_selected:
1121 continue
1122 base_sizes = self._base_board_dict[target].sizes
1123 outcome = board_dict[target]
1124 sizes = outcome.sizes
1125
1126 # Loop through the list of images, creating a dict of size
1127 # changes for each image/part. We end up with something like
1128 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1129 # which means that U-Boot data increased by 5 bytes and SPL
1130 # text decreased by 4.
1131 err = {'_target' : target}
1132 for image in sizes:
1133 if image in base_sizes:
1134 base_image = base_sizes[image]
1135 # Loop through the text, data, bss parts
1136 for part in sorted(sizes[image]):
1137 diff = sizes[image][part] - base_image[part]
1138 col = None
1139 if diff:
1140 if image == 'u-boot':
1141 name = part
1142 else:
1143 name = image + ':' + part
1144 err[name] = diff
1145 arch = board_selected[target].arch
1146 if not arch in arch_count:
1147 arch_count[arch] = 1
1148 else:
1149 arch_count[arch] += 1
1150 if not sizes:
1151 pass # Only add to our list when we have some stats
1152 elif not arch in arch_list:
1153 arch_list[arch] = [err]
1154 else:
1155 arch_list[arch].append(err)
1156
1157 # We now have a list of image size changes sorted by arch
1158 # Print out a summary of these
Simon Glassc78ed662019-10-31 07:42:53 -06001159 for arch, target_list in arch_list.items():
Simon Glassc05694f2013-04-03 11:07:16 +00001160 # Get total difference for each type
1161 totals = {}
1162 for result in target_list:
1163 total = 0
Simon Glassc78ed662019-10-31 07:42:53 -06001164 for name, diff in result.items():
Simon Glassc05694f2013-04-03 11:07:16 +00001165 if name.startswith('_'):
1166 continue
1167 total += diff
1168 if name in totals:
1169 totals[name] += diff
1170 else:
1171 totals[name] = diff
1172 result['_total_diff'] = total
1173 result['_outcome'] = board_dict[result['_target']]
1174
1175 count = len(target_list)
1176 printed_arch = False
1177 for name in sorted(totals):
1178 diff = totals[name]
1179 if diff:
1180 # Display the average difference in this name for this
1181 # architecture
1182 avg_diff = float(diff) / count
1183 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1184 msg = ' %s %+1.1f' % (name, avg_diff)
1185 if not printed_arch:
Simon Glass02811582022-01-29 14:14:18 -07001186 tprint('%10s: (for %d/%d boards)' % (arch, count,
Simon Glass4433aa92014-09-05 19:00:07 -06001187 arch_count[arch]), newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +00001188 printed_arch = True
Simon Glass02811582022-01-29 14:14:18 -07001189 tprint(msg, colour=color, newline=False)
Simon Glassc05694f2013-04-03 11:07:16 +00001190
1191 if printed_arch:
Simon Glass02811582022-01-29 14:14:18 -07001192 tprint()
Simon Glassc05694f2013-04-03 11:07:16 +00001193 if show_detail:
Simon Glassbc74d942023-07-19 17:49:06 -06001194 self.print_size_detail(target_list, show_bloat)
Simon Glassc05694f2013-04-03 11:07:16 +00001195
1196
Simon Glassbc74d942023-07-19 17:49:06 -06001197 def print_result_summary(self, board_selected, board_dict, err_lines,
Simon Glass03749d42014-08-28 09:43:44 -06001198 err_line_boards, warn_lines, warn_line_boards,
Alex Kiernan4059e302018-05-31 04:48:34 +00001199 config, environment, show_sizes, show_detail,
1200 show_bloat, show_config, show_environment):
Simon Glassc05694f2013-04-03 11:07:16 +00001201 """Compare results with the base results and display delta.
1202
1203 Only boards mentioned in board_selected will be considered. This
1204 function is intended to be called repeatedly with the results of
1205 each commit. It therefore shows a 'diff' between what it saw in
1206 the last call and what it sees now.
1207
1208 Args:
1209 board_selected: Dict containing boards to summarise, keyed by
1210 board.target
1211 board_dict: Dict containing boards for which we built this
1212 commit, keyed by board.target. The value is an Outcome object.
1213 err_lines: A list of errors for this commit, or [] if there is
1214 none, or we don't want to print errors
Simon Glass3394c9f2014-08-28 09:43:43 -06001215 err_line_boards: Dict keyed by error line, containing a list of
1216 the Board objects with that error
Simon Glass03749d42014-08-28 09:43:44 -06001217 warn_lines: A list of warnings for this commit, or [] if there is
1218 none, or we don't want to print errors
1219 warn_line_boards: Dict keyed by warning line, containing a list of
1220 the Board objects with that warning
Simon Glassdb17fb82015-02-05 22:06:15 -07001221 config: Dictionary keyed by filename - e.g. '.config'. Each
1222 value is itself a dictionary:
1223 key: config name
1224 value: config value
Alex Kiernan4059e302018-05-31 04:48:34 +00001225 environment: Dictionary keyed by environment variable, Each
1226 value is the value of environment variable.
Simon Glassc05694f2013-04-03 11:07:16 +00001227 show_sizes: Show image size deltas
Simon Glassb4002462020-03-18 09:42:43 -06001228 show_detail: Show size delta detail for each board if show_sizes
Simon Glassc05694f2013-04-03 11:07:16 +00001229 show_bloat: Show detail for each function
Simon Glassdb17fb82015-02-05 22:06:15 -07001230 show_config: Show config changes
Alex Kiernan4059e302018-05-31 04:48:34 +00001231 show_environment: Show environment changes
Simon Glassc05694f2013-04-03 11:07:16 +00001232 """
Simon Glassbc74d942023-07-19 17:49:06 -06001233 def _board_list(line, line_boards):
Simon Glass3394c9f2014-08-28 09:43:43 -06001234 """Helper function to get a line of boards containing a line
1235
1236 Args:
1237 line: Error line to search for
Simon Glassde0fefc2020-04-09 15:08:36 -06001238 line_boards: boards to search, each a Board
Simon Glass3394c9f2014-08-28 09:43:43 -06001239 Return:
Simon Glassde0fefc2020-04-09 15:08:36 -06001240 List of boards with that error line, or [] if the user has not
1241 requested such a list
Simon Glass3394c9f2014-08-28 09:43:43 -06001242 """
Simon Glass5df45222022-07-11 19:04:00 -06001243 brds = []
Simon Glassde0fefc2020-04-09 15:08:36 -06001244 board_set = set()
Simon Glass3394c9f2014-08-28 09:43:43 -06001245 if self._list_error_boards:
Simon Glass8132f982022-07-11 19:03:57 -06001246 for brd in line_boards[line]:
1247 if not brd in board_set:
Simon Glass5df45222022-07-11 19:04:00 -06001248 brds.append(brd)
Simon Glass8132f982022-07-11 19:03:57 -06001249 board_set.add(brd)
Simon Glass5df45222022-07-11 19:04:00 -06001250 return brds
Simon Glass3394c9f2014-08-28 09:43:43 -06001251
Simon Glassbc74d942023-07-19 17:49:06 -06001252 def _calc_error_delta(base_lines, base_line_boards, lines, line_boards,
Simon Glass03749d42014-08-28 09:43:44 -06001253 char):
Simon Glassde0fefc2020-04-09 15:08:36 -06001254 """Calculate the required output based on changes in errors
1255
1256 Args:
1257 base_lines: List of errors/warnings for previous commit
1258 base_line_boards: Dict keyed by error line, containing a list
1259 of the Board objects with that error in the previous commit
1260 lines: List of errors/warning for this commit, each a str
1261 line_boards: Dict keyed by error line, containing a list
1262 of the Board objects with that error in this commit
1263 char: Character representing error ('') or warning ('w'). The
1264 broken ('+') or fixed ('-') characters are added in this
1265 function
1266
1267 Returns:
1268 Tuple
1269 List of ErrLine objects for 'better' lines
1270 List of ErrLine objects for 'worse' lines
1271 """
Simon Glass03749d42014-08-28 09:43:44 -06001272 better_lines = []
1273 worse_lines = []
1274 for line in lines:
1275 if line not in base_lines:
Simon Glassbc74d942023-07-19 17:49:06 -06001276 errline = ErrLine(char + '+', _board_list(line, line_boards),
Simon Glassde0fefc2020-04-09 15:08:36 -06001277 line)
1278 worse_lines.append(errline)
Simon Glass03749d42014-08-28 09:43:44 -06001279 for line in base_lines:
1280 if line not in lines:
Simon Glassde0fefc2020-04-09 15:08:36 -06001281 errline = ErrLine(char + '-',
Simon Glassbc74d942023-07-19 17:49:06 -06001282 _board_list(line, base_line_boards), line)
Simon Glassde0fefc2020-04-09 15:08:36 -06001283 better_lines.append(errline)
Simon Glass03749d42014-08-28 09:43:44 -06001284 return better_lines, worse_lines
1285
Simon Glassbc74d942023-07-19 17:49:06 -06001286 def _calc_config(delta, name, config):
Simon Glassdb17fb82015-02-05 22:06:15 -07001287 """Calculate configuration changes
1288
1289 Args:
1290 delta: Type of the delta, e.g. '+'
1291 name: name of the file which changed (e.g. .config)
1292 config: configuration change dictionary
1293 key: config name
1294 value: config value
1295 Returns:
1296 String containing the configuration changes which can be
1297 printed
1298 """
1299 out = ''
1300 for key in sorted(config.keys()):
1301 out += '%s=%s ' % (key, config[key])
Simon Glasscad8abf2015-08-25 21:52:14 -06001302 return '%s %s: %s' % (delta, name, out)
Simon Glassdb17fb82015-02-05 22:06:15 -07001303
Simon Glassbc74d942023-07-19 17:49:06 -06001304 def _add_config(lines, name, config_plus, config_minus, config_change):
Simon Glasscad8abf2015-08-25 21:52:14 -06001305 """Add changes in configuration to a list
Simon Glassdb17fb82015-02-05 22:06:15 -07001306
1307 Args:
Simon Glasscad8abf2015-08-25 21:52:14 -06001308 lines: list to add to
1309 name: config file name
Simon Glassdb17fb82015-02-05 22:06:15 -07001310 config_plus: configurations added, dictionary
1311 key: config name
1312 value: config value
1313 config_minus: configurations removed, dictionary
1314 key: config name
1315 value: config value
1316 config_change: configurations changed, dictionary
1317 key: config name
1318 value: config value
1319 """
1320 if config_plus:
Simon Glassbc74d942023-07-19 17:49:06 -06001321 lines.append(_calc_config('+', name, config_plus))
Simon Glassdb17fb82015-02-05 22:06:15 -07001322 if config_minus:
Simon Glassbc74d942023-07-19 17:49:06 -06001323 lines.append(_calc_config('-', name, config_minus))
Simon Glassdb17fb82015-02-05 22:06:15 -07001324 if config_change:
Simon Glassbc74d942023-07-19 17:49:06 -06001325 lines.append(_calc_config('c', name, config_change))
Simon Glasscad8abf2015-08-25 21:52:14 -06001326
Simon Glassbc74d942023-07-19 17:49:06 -06001327 def _output_config_info(lines):
Simon Glasscad8abf2015-08-25 21:52:14 -06001328 for line in lines:
1329 if not line:
1330 continue
1331 if line[0] == '+':
1332 col = self.col.GREEN
1333 elif line[0] == '-':
1334 col = self.col.RED
1335 elif line[0] == 'c':
1336 col = self.col.YELLOW
Simon Glass02811582022-01-29 14:14:18 -07001337 tprint(' ' + line, newline=True, colour=col)
Simon Glasscad8abf2015-08-25 21:52:14 -06001338
Simon Glassbc74d942023-07-19 17:49:06 -06001339 def _output_err_lines(err_lines, colour):
Simon Glassac500222020-04-09 15:08:28 -06001340 """Output the line of error/warning lines, if not empty
1341
1342 Also increments self._error_lines if err_lines not empty
1343
1344 Args:
Simon Glassde0fefc2020-04-09 15:08:36 -06001345 err_lines: List of ErrLine objects, each an error or warning
1346 line, possibly including a list of boards with that
1347 error/warning
Simon Glassac500222020-04-09 15:08:28 -06001348 colour: Colour to use for output
1349 """
1350 if err_lines:
Simon Glassea49f9b2020-04-09 15:08:37 -06001351 out_list = []
Simon Glassde0fefc2020-04-09 15:08:36 -06001352 for line in err_lines:
Simon Glass5df45222022-07-11 19:04:00 -06001353 names = [brd.target for brd in line.brds]
Simon Glass070589b2020-04-09 15:08:38 -06001354 board_str = ' '.join(names) if names else ''
Simon Glassea49f9b2020-04-09 15:08:37 -06001355 if board_str:
Simon Glassf45d3742022-01-29 14:14:17 -07001356 out = self.col.build(colour, line.char + '(')
1357 out += self.col.build(self.col.MAGENTA, board_str,
Simon Glassea49f9b2020-04-09 15:08:37 -06001358 bright=False)
Simon Glassf45d3742022-01-29 14:14:17 -07001359 out += self.col.build(colour, ') %s' % line.errline)
Simon Glassea49f9b2020-04-09 15:08:37 -06001360 else:
Simon Glassf45d3742022-01-29 14:14:17 -07001361 out = self.col.build(colour, line.char + line.errline)
Simon Glassea49f9b2020-04-09 15:08:37 -06001362 out_list.append(out)
Simon Glass02811582022-01-29 14:14:18 -07001363 tprint('\n'.join(out_list))
Simon Glassac500222020-04-09 15:08:28 -06001364 self._error_lines += 1
1365
Simon Glassdb17fb82015-02-05 22:06:15 -07001366
Simon Glass454507f2018-11-06 16:02:12 -07001367 ok_boards = [] # List of boards fixed since last commit
Simon Glass071a1782018-11-06 16:02:13 -07001368 warn_boards = [] # List of boards with warnings since last commit
Simon Glass454507f2018-11-06 16:02:12 -07001369 err_boards = [] # List of new broken boards since last commit
1370 new_boards = [] # List of boards that didn't exist last time
1371 unknown_boards = [] # List of boards that were not built
Simon Glassc05694f2013-04-03 11:07:16 +00001372
1373 for target in board_dict:
1374 if target not in board_selected:
1375 continue
1376
1377 # If the board was built last time, add its outcome to a list
1378 if target in self._base_board_dict:
1379 base_outcome = self._base_board_dict[target].rc
1380 outcome = board_dict[target]
1381 if outcome.rc == OUTCOME_UNKNOWN:
Simon Glass454507f2018-11-06 16:02:12 -07001382 unknown_boards.append(target)
Simon Glassc05694f2013-04-03 11:07:16 +00001383 elif outcome.rc < base_outcome:
Simon Glass071a1782018-11-06 16:02:13 -07001384 if outcome.rc == OUTCOME_WARNING:
1385 warn_boards.append(target)
1386 else:
1387 ok_boards.append(target)
Simon Glassc05694f2013-04-03 11:07:16 +00001388 elif outcome.rc > base_outcome:
Simon Glass071a1782018-11-06 16:02:13 -07001389 if outcome.rc == OUTCOME_WARNING:
1390 warn_boards.append(target)
1391 else:
1392 err_boards.append(target)
Simon Glassc05694f2013-04-03 11:07:16 +00001393 else:
Simon Glass454507f2018-11-06 16:02:12 -07001394 new_boards.append(target)
Simon Glassc05694f2013-04-03 11:07:16 +00001395
Simon Glassac500222020-04-09 15:08:28 -06001396 # Get a list of errors and warnings that have appeared, and disappeared
Simon Glassbc74d942023-07-19 17:49:06 -06001397 better_err, worse_err = _calc_error_delta(self._base_err_lines,
Simon Glass03749d42014-08-28 09:43:44 -06001398 self._base_err_line_boards, err_lines, err_line_boards, '')
Simon Glassbc74d942023-07-19 17:49:06 -06001399 better_warn, worse_warn = _calc_error_delta(self._base_warn_lines,
Simon Glass03749d42014-08-28 09:43:44 -06001400 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassc05694f2013-04-03 11:07:16 +00001401
Simon Glass6c435622022-07-11 19:03:56 -06001402 # For the IDE mode, print out all the output
1403 if self._ide:
1404 outcome = board_dict[target]
1405 for line in outcome.err_lines:
1406 sys.stderr.write(line)
1407
Simon Glassc05694f2013-04-03 11:07:16 +00001408 # Display results by arch
Simon Glass6c435622022-07-11 19:03:56 -06001409 elif any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
Simon Glass071a1782018-11-06 16:02:13 -07001410 worse_err, better_err, worse_warn, better_warn)):
Simon Glassc05694f2013-04-03 11:07:16 +00001411 arch_list = {}
Simon Glassbc74d942023-07-19 17:49:06 -06001412 self.add_outcome(board_selected, arch_list, ok_boards, '',
Simon Glassc05694f2013-04-03 11:07:16 +00001413 self.col.GREEN)
Simon Glassbc74d942023-07-19 17:49:06 -06001414 self.add_outcome(board_selected, arch_list, warn_boards, 'w+',
Simon Glass071a1782018-11-06 16:02:13 -07001415 self.col.YELLOW)
Simon Glassbc74d942023-07-19 17:49:06 -06001416 self.add_outcome(board_selected, arch_list, err_boards, '+',
Simon Glassc05694f2013-04-03 11:07:16 +00001417 self.col.RED)
Simon Glassbc74d942023-07-19 17:49:06 -06001418 self.add_outcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
Simon Glassc05694f2013-04-03 11:07:16 +00001419 if self._show_unknown:
Simon Glassbc74d942023-07-19 17:49:06 -06001420 self.add_outcome(board_selected, arch_list, unknown_boards, '?',
Simon Glassc05694f2013-04-03 11:07:16 +00001421 self.col.MAGENTA)
Simon Glassc78ed662019-10-31 07:42:53 -06001422 for arch, target_list in arch_list.items():
Simon Glass02811582022-01-29 14:14:18 -07001423 tprint('%10s: %s' % (arch, target_list))
Simon Glassbb4dffb2014-08-09 15:33:06 -06001424 self._error_lines += 1
Simon Glassbc74d942023-07-19 17:49:06 -06001425 _output_err_lines(better_err, colour=self.col.GREEN)
1426 _output_err_lines(worse_err, colour=self.col.RED)
1427 _output_err_lines(better_warn, colour=self.col.CYAN)
1428 _output_err_lines(worse_warn, colour=self.col.YELLOW)
Simon Glassc05694f2013-04-03 11:07:16 +00001429
1430 if show_sizes:
Simon Glassbc74d942023-07-19 17:49:06 -06001431 self.print_size_summary(board_selected, board_dict, show_detail,
Simon Glassc05694f2013-04-03 11:07:16 +00001432 show_bloat)
1433
Alex Kiernan4059e302018-05-31 04:48:34 +00001434 if show_environment and self._base_environment:
1435 lines = []
1436
1437 for target in board_dict:
1438 if target not in board_selected:
1439 continue
1440
1441 tbase = self._base_environment[target]
1442 tenvironment = environment[target]
1443 environment_plus = {}
1444 environment_minus = {}
1445 environment_change = {}
1446 base = tbase.environment
Simon Glassc78ed662019-10-31 07:42:53 -06001447 for key, value in tenvironment.environment.items():
Alex Kiernan4059e302018-05-31 04:48:34 +00001448 if key not in base:
1449 environment_plus[key] = value
Simon Glassc78ed662019-10-31 07:42:53 -06001450 for key, value in base.items():
Alex Kiernan4059e302018-05-31 04:48:34 +00001451 if key not in tenvironment.environment:
1452 environment_minus[key] = value
Simon Glassc78ed662019-10-31 07:42:53 -06001453 for key, value in base.items():
Alex Kiernan4059e302018-05-31 04:48:34 +00001454 new_value = tenvironment.environment.get(key)
1455 if new_value and value != new_value:
1456 desc = '%s -> %s' % (value, new_value)
1457 environment_change[key] = desc
1458
Simon Glassbc74d942023-07-19 17:49:06 -06001459 _add_config(lines, target, environment_plus, environment_minus,
Alex Kiernan4059e302018-05-31 04:48:34 +00001460 environment_change)
1461
Simon Glassbc74d942023-07-19 17:49:06 -06001462 _output_config_info(lines)
Alex Kiernan4059e302018-05-31 04:48:34 +00001463
Simon Glasscad8abf2015-08-25 21:52:14 -06001464 if show_config and self._base_config:
1465 summary = {}
1466 arch_config_plus = {}
1467 arch_config_minus = {}
1468 arch_config_change = {}
1469 arch_list = []
1470
1471 for target in board_dict:
1472 if target not in board_selected:
1473 continue
1474 arch = board_selected[target].arch
1475 if arch not in arch_list:
1476 arch_list.append(arch)
1477
1478 for arch in arch_list:
1479 arch_config_plus[arch] = {}
1480 arch_config_minus[arch] = {}
1481 arch_config_change[arch] = {}
Simon Glasscde5c302016-11-13 14:25:53 -07001482 for name in self.config_filenames:
Simon Glasscad8abf2015-08-25 21:52:14 -06001483 arch_config_plus[arch][name] = {}
1484 arch_config_minus[arch][name] = {}
1485 arch_config_change[arch][name] = {}
1486
1487 for target in board_dict:
1488 if target not in board_selected:
Simon Glassdb17fb82015-02-05 22:06:15 -07001489 continue
Simon Glasscad8abf2015-08-25 21:52:14 -06001490
1491 arch = board_selected[target].arch
1492
1493 all_config_plus = {}
1494 all_config_minus = {}
1495 all_config_change = {}
1496 tbase = self._base_config[target]
1497 tconfig = config[target]
1498 lines = []
Simon Glasscde5c302016-11-13 14:25:53 -07001499 for name in self.config_filenames:
Simon Glasscad8abf2015-08-25 21:52:14 -06001500 if not tconfig.config[name]:
1501 continue
1502 config_plus = {}
1503 config_minus = {}
1504 config_change = {}
1505 base = tbase.config[name]
Simon Glassc78ed662019-10-31 07:42:53 -06001506 for key, value in tconfig.config[name].items():
Simon Glasscad8abf2015-08-25 21:52:14 -06001507 if key not in base:
1508 config_plus[key] = value
1509 all_config_plus[key] = value
Simon Glassc78ed662019-10-31 07:42:53 -06001510 for key, value in base.items():
Simon Glasscad8abf2015-08-25 21:52:14 -06001511 if key not in tconfig.config[name]:
1512 config_minus[key] = value
1513 all_config_minus[key] = value
Simon Glassc78ed662019-10-31 07:42:53 -06001514 for key, value in base.items():
Simon Glasscad8abf2015-08-25 21:52:14 -06001515 new_value = tconfig.config.get(key)
1516 if new_value and value != new_value:
1517 desc = '%s -> %s' % (value, new_value)
1518 config_change[key] = desc
1519 all_config_change[key] = desc
1520
1521 arch_config_plus[arch][name].update(config_plus)
1522 arch_config_minus[arch][name].update(config_minus)
1523 arch_config_change[arch][name].update(config_change)
1524
Simon Glassbc74d942023-07-19 17:49:06 -06001525 _add_config(lines, name, config_plus, config_minus,
Simon Glasscad8abf2015-08-25 21:52:14 -06001526 config_change)
Simon Glassbc74d942023-07-19 17:49:06 -06001527 _add_config(lines, 'all', all_config_plus, all_config_minus,
Simon Glasscad8abf2015-08-25 21:52:14 -06001528 all_config_change)
1529 summary[target] = '\n'.join(lines)
1530
1531 lines_by_target = {}
Simon Glassc78ed662019-10-31 07:42:53 -06001532 for target, lines in summary.items():
Simon Glasscad8abf2015-08-25 21:52:14 -06001533 if lines in lines_by_target:
1534 lines_by_target[lines].append(target)
1535 else:
1536 lines_by_target[lines] = [target]
1537
1538 for arch in arch_list:
1539 lines = []
1540 all_plus = {}
1541 all_minus = {}
1542 all_change = {}
Simon Glasscde5c302016-11-13 14:25:53 -07001543 for name in self.config_filenames:
Simon Glasscad8abf2015-08-25 21:52:14 -06001544 all_plus.update(arch_config_plus[arch][name])
1545 all_minus.update(arch_config_minus[arch][name])
1546 all_change.update(arch_config_change[arch][name])
Simon Glassbc74d942023-07-19 17:49:06 -06001547 _add_config(lines, name, arch_config_plus[arch][name],
Simon Glasscad8abf2015-08-25 21:52:14 -06001548 arch_config_minus[arch][name],
1549 arch_config_change[arch][name])
Simon Glassbc74d942023-07-19 17:49:06 -06001550 _add_config(lines, 'all', all_plus, all_minus, all_change)
Simon Glasscad8abf2015-08-25 21:52:14 -06001551 #arch_summary[target] = '\n'.join(lines)
1552 if lines:
Simon Glass02811582022-01-29 14:14:18 -07001553 tprint('%s:' % arch)
Simon Glassbc74d942023-07-19 17:49:06 -06001554 _output_config_info(lines)
Simon Glasscad8abf2015-08-25 21:52:14 -06001555
Simon Glassc78ed662019-10-31 07:42:53 -06001556 for lines, targets in lines_by_target.items():
Simon Glasscad8abf2015-08-25 21:52:14 -06001557 if not lines:
1558 continue
Simon Glass02811582022-01-29 14:14:18 -07001559 tprint('%s :' % ' '.join(sorted(targets)))
Simon Glassbc74d942023-07-19 17:49:06 -06001560 _output_config_info(lines.split('\n'))
Simon Glasscad8abf2015-08-25 21:52:14 -06001561
Simon Glassdb17fb82015-02-05 22:06:15 -07001562
Simon Glassc05694f2013-04-03 11:07:16 +00001563 # Save our updated information for the next call to this function
1564 self._base_board_dict = board_dict
1565 self._base_err_lines = err_lines
Simon Glass03749d42014-08-28 09:43:44 -06001566 self._base_warn_lines = warn_lines
1567 self._base_err_line_boards = err_line_boards
1568 self._base_warn_line_boards = warn_line_boards
Simon Glassdb17fb82015-02-05 22:06:15 -07001569 self._base_config = config
Alex Kiernan4059e302018-05-31 04:48:34 +00001570 self._base_environment = environment
Simon Glassc05694f2013-04-03 11:07:16 +00001571
1572 # Get a list of boards that did not get built, if needed
1573 not_built = []
Simon Glass8132f982022-07-11 19:03:57 -06001574 for brd in board_selected:
1575 if not brd in board_dict:
1576 not_built.append(brd)
Simon Glassc05694f2013-04-03 11:07:16 +00001577 if not_built:
Simon Glass02811582022-01-29 14:14:18 -07001578 tprint("Boards not built (%d): %s" % (len(not_built),
Simon Glass4433aa92014-09-05 19:00:07 -06001579 ', '.join(not_built)))
Simon Glassc05694f2013-04-03 11:07:16 +00001580
Simon Glassbc74d942023-07-19 17:49:06 -06001581 def produce_result_summary(self, commit_upto, commits, board_selected):
Simon Glass03749d42014-08-28 09:43:44 -06001582 (board_dict, err_lines, err_line_boards, warn_lines,
Simon Glassbc74d942023-07-19 17:49:06 -06001583 warn_line_boards, config, environment) = self.get_result_summary(
Simon Glass3394c9f2014-08-28 09:43:43 -06001584 board_selected, commit_upto,
Simon Glassdb17fb82015-02-05 22:06:15 -07001585 read_func_sizes=self._show_bloat,
Alex Kiernan4059e302018-05-31 04:48:34 +00001586 read_config=self._show_config,
1587 read_environment=self._show_environment)
Simon Glasseb48bbc2014-08-09 15:33:02 -06001588 if commits:
1589 msg = '%02d: %s' % (commit_upto + 1,
1590 commits[commit_upto].subject)
Simon Glass02811582022-01-29 14:14:18 -07001591 tprint(msg, colour=self.col.BLUE)
Simon Glassbc74d942023-07-19 17:49:06 -06001592 self.print_result_summary(board_selected, board_dict,
Simon Glass3394c9f2014-08-28 09:43:43 -06001593 err_lines if self._show_errors else [], err_line_boards,
Simon Glass03749d42014-08-28 09:43:44 -06001594 warn_lines if self._show_errors else [], warn_line_boards,
Alex Kiernan4059e302018-05-31 04:48:34 +00001595 config, environment, self._show_sizes, self._show_detail,
1596 self._show_bloat, self._show_config, self._show_environment)
Simon Glassc05694f2013-04-03 11:07:16 +00001597
Simon Glassbc74d942023-07-19 17:49:06 -06001598 def show_summary(self, commits, board_selected):
Simon Glassc05694f2013-04-03 11:07:16 +00001599 """Show a build summary for U-Boot for a given board list.
1600
1601 Reset the result summary, then repeatedly call GetResultSummary on
1602 each commit's results, then display the differences we see.
1603
1604 Args:
1605 commit: Commit objects to summarise
1606 board_selected: Dict containing boards to summarise
Simon Glassc05694f2013-04-03 11:07:16 +00001607 """
Simon Glassd326ad72014-08-09 15:32:59 -06001608 self.commit_count = len(commits) if commits else 1
Simon Glassc05694f2013-04-03 11:07:16 +00001609 self.commits = commits
Simon Glassbc74d942023-07-19 17:49:06 -06001610 self.reset_result_summary(board_selected)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001611 self._error_lines = 0
Simon Glassc05694f2013-04-03 11:07:16 +00001612
1613 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassbc74d942023-07-19 17:49:06 -06001614 self.produce_result_summary(commit_upto, commits, board_selected)
Simon Glassbb4dffb2014-08-09 15:33:06 -06001615 if not self._error_lines:
Simon Glass02811582022-01-29 14:14:18 -07001616 tprint('(no errors to report)', colour=self.col.GREEN)
Simon Glassc05694f2013-04-03 11:07:16 +00001617
1618
Simon Glassbc74d942023-07-19 17:49:06 -06001619 def setup_build(self, board_selected, commits):
Simon Glassc05694f2013-04-03 11:07:16 +00001620 """Set up ready to start a build.
1621
1622 Args:
1623 board_selected: Selected boards to build
1624 commits: Selected commits to build
1625 """
1626 # First work out how many commits we will build
Simon Glassc78ed662019-10-31 07:42:53 -06001627 count = (self.commit_count + self._step - 1) // self._step
Simon Glassc05694f2013-04-03 11:07:16 +00001628 self.count = len(board_selected) * count
1629 self.upto = self.warned = self.fail = 0
1630 self._timestamps = collections.deque()
1631
Simon Glassbc74d942023-07-19 17:49:06 -06001632 def get_thread_dir(self, thread_num):
Simon Glassc05694f2013-04-03 11:07:16 +00001633 """Get the directory path to the working dir for a thread.
1634
1635 Args:
Simon Glassc635d892021-01-30 22:17:46 -07001636 thread_num: Number of thread to check (-1 for main process, which
1637 is treated as 0)
Simon Glassc05694f2013-04-03 11:07:16 +00001638 """
Simon Glassb6eb8cf2020-03-18 09:42:42 -06001639 if self.work_in_output:
1640 return self._working_dir
Simon Glassc635d892021-01-30 22:17:46 -07001641 return os.path.join(self._working_dir, '%02d' % max(thread_num, 0))
Simon Glassc05694f2013-04-03 11:07:16 +00001642
Simon Glassbc74d942023-07-19 17:49:06 -06001643 def _prepare_thread(self, thread_num, setup_git):
Simon Glassc05694f2013-04-03 11:07:16 +00001644 """Prepare the working directory for a thread.
1645
1646 This clones or fetches the repo into the thread's work directory.
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001647 Optionally, it can create a linked working tree of the repo in the
1648 thread's work directory instead.
Simon Glassc05694f2013-04-03 11:07:16 +00001649
1650 Args:
1651 thread_num: Thread number (0, 1, ...)
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001652 setup_git:
1653 'clone' to set up a git clone
1654 'worktree' to set up a git worktree
Simon Glassc05694f2013-04-03 11:07:16 +00001655 """
Simon Glassbc74d942023-07-19 17:49:06 -06001656 thread_dir = self.get_thread_dir(thread_num)
Simon Glassc5077c32023-07-19 17:49:08 -06001657 builderthread.mkdir(thread_dir)
Simon Glassc05694f2013-04-03 11:07:16 +00001658 git_dir = os.path.join(thread_dir, '.git')
1659
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001660 # Create a worktree or a git repo clone for this thread if it
1661 # doesn't already exist
Simon Glassd326ad72014-08-09 15:32:59 -06001662 if setup_git and self.git_dir:
Simon Glassc05694f2013-04-03 11:07:16 +00001663 src_dir = os.path.abspath(self.git_dir)
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001664 if os.path.isdir(git_dir):
1665 # This is a clone of the src_dir repo, we can keep using
1666 # it but need to fetch from src_dir.
Simon Glass02811582022-01-29 14:14:18 -07001667 tprint('\rFetching repo for thread %d' % thread_num,
Simon Glass43054932020-04-09 15:08:43 -06001668 newline=False)
Simon Glass761648b2022-01-29 14:14:11 -07001669 gitutil.fetch(git_dir, thread_dir)
Simon Glass02811582022-01-29 14:14:18 -07001670 terminal.print_clear()
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001671 elif os.path.isfile(git_dir):
1672 # This is a worktree of the src_dir repo, we don't need to
1673 # create it again or update it in any way.
1674 pass
1675 elif os.path.exists(git_dir):
1676 # Don't know what could trigger this, but we probably
1677 # can't create a git worktree/clone here.
1678 raise ValueError('Git dir %s exists, but is not a file '
1679 'or a directory.' % git_dir)
1680 elif setup_git == 'worktree':
Simon Glass02811582022-01-29 14:14:18 -07001681 tprint('\rChecking out worktree for thread %d' % thread_num,
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001682 newline=False)
Simon Glass761648b2022-01-29 14:14:11 -07001683 gitutil.add_worktree(src_dir, thread_dir)
Simon Glass02811582022-01-29 14:14:18 -07001684 terminal.print_clear()
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001685 elif setup_git == 'clone' or setup_git == True:
Simon Glass02811582022-01-29 14:14:18 -07001686 tprint('\rCloning repo for thread %d' % thread_num,
Simon Glass2e2a6e62016-09-18 16:48:31 -06001687 newline=False)
Simon Glass761648b2022-01-29 14:14:11 -07001688 gitutil.clone(src_dir, thread_dir)
Simon Glass02811582022-01-29 14:14:18 -07001689 terminal.print_clear()
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001690 else:
1691 raise ValueError("Can't setup git repo with %s." % setup_git)
Simon Glassc05694f2013-04-03 11:07:16 +00001692
Simon Glassbc74d942023-07-19 17:49:06 -06001693 def _prepare_working_space(self, max_threads, setup_git):
Simon Glassc05694f2013-04-03 11:07:16 +00001694 """Prepare the working directory for use.
1695
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001696 Set up the git repo for each thread. Creates a linked working tree
1697 if git-worktree is available, or clones the repo if it isn't.
Simon Glassc05694f2013-04-03 11:07:16 +00001698
1699 Args:
Simon Glassc635d892021-01-30 22:17:46 -07001700 max_threads: Maximum number of threads we expect to need. If 0 then
1701 1 is set up, since the main process still needs somewhere to
1702 work
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001703 setup_git: True to set up a git worktree or a git clone
Simon Glassc05694f2013-04-03 11:07:16 +00001704 """
Simon Glassc5077c32023-07-19 17:49:08 -06001705 builderthread.mkdir(self._working_dir)
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001706 if setup_git and self.git_dir:
1707 src_dir = os.path.abspath(self.git_dir)
Simon Glass761648b2022-01-29 14:14:11 -07001708 if gitutil.check_worktree_is_available(src_dir):
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001709 setup_git = 'worktree'
1710 # If we previously added a worktree but the directory for it
1711 # got deleted, we need to prune its files from the repo so
1712 # that we can check out another in its place.
Simon Glass761648b2022-01-29 14:14:11 -07001713 gitutil.prune_worktrees(src_dir)
Alper Nebi Yasakfede44a2020-09-03 15:51:03 +03001714 else:
1715 setup_git = 'clone'
Simon Glassc635d892021-01-30 22:17:46 -07001716
1717 # Always do at least one thread
1718 for thread in range(max(max_threads, 1)):
Simon Glassbc74d942023-07-19 17:49:06 -06001719 self._prepare_thread(thread, setup_git)
Simon Glassc05694f2013-04-03 11:07:16 +00001720
Simon Glassbc74d942023-07-19 17:49:06 -06001721 def _get_output_space_removals(self):
Simon Glassc05694f2013-04-03 11:07:16 +00001722 """Get the output directories ready to receive files.
1723
Simon Glass5dc1ca72020-03-18 09:42:45 -06001724 Figure out what needs to be deleted in the output directory before it
1725 can be used. We only delete old buildman directories which have the
Simon Glass4cb54682023-07-19 17:49:10 -06001726 expected name pattern. See get_output_dir().
Simon Glass5dc1ca72020-03-18 09:42:45 -06001727
1728 Returns:
1729 List of full paths of directories to remove
Simon Glassc05694f2013-04-03 11:07:16 +00001730 """
Simon Glassb9a9b7c2014-12-01 17:33:53 -07001731 if not self.commits:
1732 return
Simon Glassc05694f2013-04-03 11:07:16 +00001733 dir_list = []
1734 for commit_upto in range(self.commit_count):
Simon Glass4cb54682023-07-19 17:49:10 -06001735 dir_list.append(self.get_output_dir(commit_upto))
Simon Glassc05694f2013-04-03 11:07:16 +00001736
Simon Glass83cb6cc2016-09-18 16:48:32 -06001737 to_remove = []
Simon Glassc05694f2013-04-03 11:07:16 +00001738 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1739 if dirname not in dir_list:
Simon Glass5dc1ca72020-03-18 09:42:45 -06001740 leaf = dirname[len(self.base_dir) + 1:]
Ovidiu Panaitee8e9cb2020-05-15 09:30:12 +03001741 m = re.match('[0-9]+_g[0-9a-f]+_.*', leaf)
Simon Glass5dc1ca72020-03-18 09:42:45 -06001742 if m:
1743 to_remove.append(dirname)
1744 return to_remove
1745
Simon Glassbc74d942023-07-19 17:49:06 -06001746 def _prepare_output_space(self):
Simon Glass5dc1ca72020-03-18 09:42:45 -06001747 """Get the output directories ready to receive files.
1748
1749 We delete any output directories which look like ones we need to
1750 create. Having left over directories is confusing when the user wants
1751 to check the output manually.
1752 """
Simon Glassbc74d942023-07-19 17:49:06 -06001753 to_remove = self._get_output_space_removals()
Simon Glass83cb6cc2016-09-18 16:48:32 -06001754 if to_remove:
Simon Glass02811582022-01-29 14:14:18 -07001755 tprint('Removing %d old build directories...' % len(to_remove),
Simon Glass83cb6cc2016-09-18 16:48:32 -06001756 newline=False)
1757 for dirname in to_remove:
Simon Glassc05694f2013-04-03 11:07:16 +00001758 shutil.rmtree(dirname)
Simon Glass02811582022-01-29 14:14:18 -07001759 terminal.print_clear()
Simon Glassc05694f2013-04-03 11:07:16 +00001760
Simon Glassbc74d942023-07-19 17:49:06 -06001761 def build_boards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassc05694f2013-04-03 11:07:16 +00001762 """Build all commits for a list of boards
1763
1764 Args:
1765 commits: List of commits to be build, each a Commit object
1766 boards_selected: Dict of selected boards, key is target name,
1767 value is Board object
Simon Glassc05694f2013-04-03 11:07:16 +00001768 keep_outputs: True to save build output files
Simon Glass78e418e2014-08-09 15:33:03 -06001769 verbose: Display build results as they are completed
Simon Glassc2f91072014-08-28 09:43:39 -06001770 Returns:
1771 Tuple containing:
1772 - number of boards that failed to build
1773 - number of boards that issued warnings
Simon Glass9bf9a722021-04-11 16:27:27 +12001774 - list of thread exceptions raised
Simon Glassc05694f2013-04-03 11:07:16 +00001775 """
Simon Glassd326ad72014-08-09 15:32:59 -06001776 self.commit_count = len(commits) if commits else 1
Simon Glassc05694f2013-04-03 11:07:16 +00001777 self.commits = commits
Simon Glass78e418e2014-08-09 15:33:03 -06001778 self._verbose = verbose
Simon Glassc05694f2013-04-03 11:07:16 +00001779
Simon Glassbc74d942023-07-19 17:49:06 -06001780 self.reset_result_summary(board_selected)
Simon Glassc5077c32023-07-19 17:49:08 -06001781 builderthread.mkdir(self.base_dir, parents = True)
Simon Glassbc74d942023-07-19 17:49:06 -06001782 self._prepare_working_space(min(self.num_threads, len(board_selected)),
Simon Glassd326ad72014-08-09 15:32:59 -06001783 commits is not None)
Simon Glassbc74d942023-07-19 17:49:06 -06001784 self._prepare_output_space()
Simon Glass6c435622022-07-11 19:03:56 -06001785 if not self._ide:
1786 tprint('\rStarting build...', newline=False)
Simon Glass7190a172023-09-07 10:00:19 -06001787 self._start_time = datetime.now()
Simon Glassbc74d942023-07-19 17:49:06 -06001788 self.setup_build(board_selected, commits)
1789 self.process_result(None)
Simon Glass9bf9a722021-04-11 16:27:27 +12001790 self.thread_exceptions = []
Simon Glassc05694f2013-04-03 11:07:16 +00001791 # Create jobs to build all commits for each board
Simon Glassc78ed662019-10-31 07:42:53 -06001792 for brd in board_selected.values():
Simon Glass4a1e88b2014-08-09 15:33:00 -06001793 job = builderthread.BuilderJob()
Simon Glass8132f982022-07-11 19:03:57 -06001794 job.brd = brd
Simon Glassc05694f2013-04-03 11:07:16 +00001795 job.commits = commits
1796 job.keep_outputs = keep_outputs
Simon Glassb6eb8cf2020-03-18 09:42:42 -06001797 job.work_in_output = self.work_in_output
Simon Glasse5650a82022-01-22 05:07:33 -07001798 job.adjust_cfg = self.adjust_cfg
Simon Glassc05694f2013-04-03 11:07:16 +00001799 job.step = self._step
Simon Glassc635d892021-01-30 22:17:46 -07001800 if self.num_threads:
1801 self.queue.put(job)
1802 else:
Simon Glassc5077c32023-07-19 17:49:08 -06001803 self._single_builder.run_job(job)
Simon Glassc05694f2013-04-03 11:07:16 +00001804
Simon Glassc635d892021-01-30 22:17:46 -07001805 if self.num_threads:
1806 term = threading.Thread(target=self.queue.join)
1807 term.setDaemon(True)
1808 term.start()
1809 while term.is_alive():
1810 term.join(100)
Simon Glassc05694f2013-04-03 11:07:16 +00001811
Simon Glassc635d892021-01-30 22:17:46 -07001812 # Wait until we have processed all output
1813 self.out_queue.join()
Simon Glass6c435622022-07-11 19:03:56 -06001814 if not self._ide:
1815 tprint()
Simon Glass726ae812020-04-09 15:08:47 -06001816
Simon Glass6c435622022-07-11 19:03:56 -06001817 msg = 'Completed: %d total built' % self.count
1818 if self.already_done:
1819 msg += ' (%d previously' % self.already_done
1820 if self.already_done != self.count:
1821 msg += ', %d newly' % (self.count - self.already_done)
1822 msg += ')'
1823 duration = datetime.now() - self._start_time
1824 if duration > timedelta(microseconds=1000000):
1825 if duration.microseconds >= 500000:
1826 duration = duration + timedelta(seconds=1)
1827 duration = duration - timedelta(microseconds=duration.microseconds)
1828 rate = float(self.count) / duration.total_seconds()
1829 msg += ', duration %s, rate %1.2f' % (duration, rate)
1830 tprint(msg)
1831 if self.thread_exceptions:
1832 tprint('Failed: %d thread exceptions' % len(self.thread_exceptions),
1833 colour=self.col.RED)
Simon Glass726ae812020-04-09 15:08:47 -06001834
Simon Glass9bf9a722021-04-11 16:27:27 +12001835 return (self.fail, self.warned, self.thread_exceptions)