Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. |
| 2 | # |
| 3 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> |
| 4 | # |
Wolfgang Denk | d79de1d | 2013-07-08 09:37:19 +0200 | [diff] [blame] | 5 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 6 | # |
| 7 | |
| 8 | import collections |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 9 | from datetime import datetime, timedelta |
| 10 | import glob |
| 11 | import os |
| 12 | import re |
| 13 | import Queue |
| 14 | import shutil |
| 15 | import string |
| 16 | import sys |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 17 | import time |
| 18 | |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 19 | import builderthread |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 20 | import command |
| 21 | import gitutil |
| 22 | import terminal |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 23 | from terminal import Print |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 24 | import toolchain |
| 25 | |
| 26 | |
| 27 | """ |
| 28 | Theory of Operation |
| 29 | |
| 30 | Please see README for user documentation, and you should be familiar with |
| 31 | that before trying to make sense of this. |
| 32 | |
| 33 | Buildman works by keeping the machine as busy as possible, building different |
| 34 | commits for different boards on multiple CPUs at once. |
| 35 | |
| 36 | The source repo (self.git_dir) contains all the commits to be built. Each |
| 37 | thread works on a single board at a time. It checks out the first commit, |
| 38 | configures it for that board, then builds it. Then it checks out the next |
| 39 | commit and builds it (typically without re-configuring). When it runs out |
| 40 | of commits, it gets another job from the builder and starts again with that |
| 41 | board. |
| 42 | |
| 43 | Clearly the builder threads could work either way - they could check out a |
| 44 | commit and then built it for all boards. Using separate directories for each |
| 45 | commit/board pair they could leave their build product around afterwards |
| 46 | also. |
| 47 | |
| 48 | The intent behind building a single board for multiple commits, is to make |
| 49 | use of incremental builds. Since each commit is built incrementally from |
| 50 | the previous one, builds are faster. Reconfiguring for a different board |
| 51 | removes all intermediate object files. |
| 52 | |
| 53 | Many threads can be working at once, but each has its own working directory. |
| 54 | When a thread finishes a build, it puts the output files into a result |
| 55 | directory. |
| 56 | |
| 57 | The base directory used by buildman is normally '../<branch>', i.e. |
| 58 | a directory higher than the source repository and named after the branch |
| 59 | being built. |
| 60 | |
| 61 | Within the base directory, we have one subdirectory for each commit. Within |
| 62 | that is one subdirectory for each board. Within that is the build output for |
| 63 | that commit/board combination. |
| 64 | |
| 65 | Buildman also create working directories for each thread, in a .bm-work/ |
| 66 | subdirectory in the base dir. |
| 67 | |
| 68 | As an example, say we are building branch 'us-net' for boards 'sandbox' and |
| 69 | 'seaboard', and say that us-net has two commits. We will have directories |
| 70 | like this: |
| 71 | |
| 72 | us-net/ base directory |
| 73 | 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ |
| 74 | sandbox/ |
| 75 | u-boot.bin |
| 76 | seaboard/ |
| 77 | u-boot.bin |
| 78 | 02_of_02_g4ed4ebc_net--Check-tftp-comp/ |
| 79 | sandbox/ |
| 80 | u-boot.bin |
| 81 | seaboard/ |
| 82 | u-boot.bin |
| 83 | .bm-work/ |
| 84 | 00/ working directory for thread 0 (contains source checkout) |
| 85 | build/ build output |
| 86 | 01/ working directory for thread 1 |
| 87 | build/ build output |
| 88 | ... |
| 89 | u-boot/ source directory |
| 90 | .git/ repository |
| 91 | """ |
| 92 | |
| 93 | # Possible build outcomes |
| 94 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) |
| 95 | |
| 96 | # Translate a commit subject into a valid filename |
| 97 | trans_valid_chars = string.maketrans("/: ", "---") |
| 98 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 99 | CONFIG_FILENAMES = [ |
| 100 | '.config', '.config-spl', '.config-tpl', |
| 101 | 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', |
| 102 | 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', |
| 103 | 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' |
| 104 | ] |
| 105 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 106 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 107 | class Builder: |
| 108 | """Class for building U-Boot for a particular commit. |
| 109 | |
| 110 | Public members: (many should ->private) |
| 111 | active: True if the builder is active and has not been stopped |
| 112 | already_done: Number of builds already completed |
| 113 | base_dir: Base directory to use for builder |
| 114 | checkout: True to check out source, False to skip that step. |
| 115 | This is used for testing. |
| 116 | col: terminal.Color() object |
| 117 | count: Number of commits to build |
| 118 | do_make: Method to call to invoke Make |
| 119 | fail: Number of builds that failed due to error |
| 120 | force_build: Force building even if a build already exists |
| 121 | force_config_on_failure: If a commit fails for a board, disable |
| 122 | incremental building for the next commit we build for that |
| 123 | board, so that we will see all warnings/errors again. |
Simon Glass | 7041c39 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 124 | force_build_failures: If a previously-built build (i.e. built on |
| 125 | a previous run of buildman) is marked as failed, rebuild it. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 126 | git_dir: Git directory containing source repository |
| 127 | last_line_len: Length of the last line we printed (used for erasing |
| 128 | it with new progress information) |
| 129 | num_jobs: Number of jobs to run at once (passed to make as -j) |
| 130 | num_threads: Number of builder threads to run |
| 131 | out_queue: Queue of results to process |
| 132 | re_make_err: Compiled regular expression for ignore_lines |
| 133 | queue: Queue of jobs to run |
| 134 | threads: List of active threads |
| 135 | toolchains: Toolchains object to use for building |
| 136 | upto: Current commit number we are building (0.count-1) |
| 137 | warned: Number of builds that produced at least one warning |
Simon Glass | f3018b7a | 2014-07-14 17:51:02 -0600 | [diff] [blame] | 138 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
| 139 | incremental building, where buildman reconfigures on the first |
| 140 | commit for a baord, and then just does an incremental build for |
| 141 | the following commits. In fact buildman will reconfigure and |
| 142 | retry for any failing commits, so generally the only effect of |
| 143 | this option is to slow things down. |
Simon Glass | 38df2e2 | 2014-07-14 17:51:03 -0600 | [diff] [blame] | 144 | in_tree: Build U-Boot in-tree instead of specifying an output |
| 145 | directory separate from the source code. This option is really |
| 146 | only useful for testing in-tree builds. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 147 | |
| 148 | Private members: |
| 149 | _base_board_dict: Last-summarised Dict of boards |
| 150 | _base_err_lines: Last-summarised list of errors |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 151 | _base_warn_lines: Last-summarised list of warnings |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 152 | _build_period_us: Time taken for a single build (float object). |
| 153 | _complete_delay: Expected delay until completion (timedelta) |
| 154 | _next_delay_update: Next time we plan to display a progress update |
| 155 | (datatime) |
| 156 | _show_unknown: Show unknown boards (those not built) in summary |
| 157 | _timestamps: List of timestamps for the completion of the last |
| 158 | last _timestamp_count builds. Each is a datetime object. |
| 159 | _timestamp_count: Number of timestamps to keep in our list. |
| 160 | _working_dir: Base working directory containing all threads |
| 161 | """ |
| 162 | class Outcome: |
| 163 | """Records a build outcome for a single make invocation |
| 164 | |
| 165 | Public Members: |
| 166 | rc: Outcome value (OUTCOME_...) |
| 167 | err_lines: List of error lines or [] if none |
| 168 | sizes: Dictionary of image size information, keyed by filename |
| 169 | - Each value is itself a dictionary containing |
| 170 | values for 'text', 'data' and 'bss', being the integer |
| 171 | size in bytes of each section. |
| 172 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each |
| 173 | value is itself a dictionary: |
| 174 | key: function name |
| 175 | value: Size of function in bytes |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 176 | config: Dictionary keyed by filename - e.g. '.config'. Each |
| 177 | value is itself a dictionary: |
| 178 | key: config name |
| 179 | value: config value |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 180 | """ |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 181 | def __init__(self, rc, err_lines, sizes, func_sizes, config): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 182 | self.rc = rc |
| 183 | self.err_lines = err_lines |
| 184 | self.sizes = sizes |
| 185 | self.func_sizes = func_sizes |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 186 | self.config = config |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 187 | |
| 188 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, |
Simon Glass | e87bde1 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 189 | gnu_make='make', checkout=True, show_unknown=True, step=1, |
Simon Glass | 655b610 | 2014-12-01 17:34:07 -0700 | [diff] [blame] | 190 | no_subdirs=False, full_path=False, verbose_build=False): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 191 | """Create a new Builder object |
| 192 | |
| 193 | Args: |
| 194 | toolchains: Toolchains object to use for building |
| 195 | base_dir: Base directory to use for builder |
| 196 | git_dir: Git directory containing source repository |
| 197 | num_threads: Number of builder threads to run |
| 198 | num_jobs: Number of jobs to run at once (passed to make as -j) |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 199 | gnu_make: the command name of GNU Make. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 200 | checkout: True to check out source, False to skip that step. |
| 201 | This is used for testing. |
| 202 | show_unknown: Show unknown boards (those not built) in summary |
| 203 | step: 1 to process every commit, n to process every nth commit |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 204 | no_subdirs: Don't create subdirectories when building current |
| 205 | source for a single board |
| 206 | full_path: Return the full path in CROSS_COMPILE and don't set |
| 207 | PATH |
Simon Glass | 655b610 | 2014-12-01 17:34:07 -0700 | [diff] [blame] | 208 | verbose_build: Run build with V=1 and don't use 'make -s' |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 209 | """ |
| 210 | self.toolchains = toolchains |
| 211 | self.base_dir = base_dir |
| 212 | self._working_dir = os.path.join(base_dir, '.bm-work') |
| 213 | self.threads = [] |
| 214 | self.active = True |
| 215 | self.do_make = self.Make |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 216 | self.gnu_make = gnu_make |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 217 | self.checkout = checkout |
| 218 | self.num_threads = num_threads |
| 219 | self.num_jobs = num_jobs |
| 220 | self.already_done = 0 |
| 221 | self.force_build = False |
| 222 | self.git_dir = git_dir |
| 223 | self._show_unknown = show_unknown |
| 224 | self._timestamp_count = 10 |
| 225 | self._build_period_us = None |
| 226 | self._complete_delay = None |
| 227 | self._next_delay_update = datetime.now() |
| 228 | self.force_config_on_failure = True |
Simon Glass | 7041c39 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 229 | self.force_build_failures = False |
Simon Glass | f3018b7a | 2014-07-14 17:51:02 -0600 | [diff] [blame] | 230 | self.force_reconfig = False |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 231 | self._step = step |
Simon Glass | 38df2e2 | 2014-07-14 17:51:03 -0600 | [diff] [blame] | 232 | self.in_tree = False |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 233 | self._error_lines = 0 |
Simon Glass | e87bde1 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 234 | self.no_subdirs = no_subdirs |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 235 | self.full_path = full_path |
Simon Glass | 655b610 | 2014-12-01 17:34:07 -0700 | [diff] [blame] | 236 | self.verbose_build = verbose_build |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 237 | |
| 238 | self.col = terminal.Color() |
| 239 | |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 240 | self._re_function = re.compile('(.*): In function.*') |
| 241 | self._re_files = re.compile('In file included from.*') |
| 242 | self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') |
| 243 | self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') |
| 244 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 245 | self.queue = Queue.Queue() |
| 246 | self.out_queue = Queue.Queue() |
| 247 | for i in range(self.num_threads): |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 248 | t = builderthread.BuilderThread(self, i) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 249 | t.setDaemon(True) |
| 250 | t.start() |
| 251 | self.threads.append(t) |
| 252 | |
| 253 | self.last_line_len = 0 |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 254 | t = builderthread.ResultThread(self) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 255 | t.setDaemon(True) |
| 256 | t.start() |
| 257 | self.threads.append(t) |
| 258 | |
| 259 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] |
| 260 | self.re_make_err = re.compile('|'.join(ignore_lines)) |
| 261 | |
| 262 | def __del__(self): |
| 263 | """Get rid of all threads created by the builder""" |
| 264 | for t in self.threads: |
| 265 | del t |
| 266 | |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 267 | def SetDisplayOptions(self, show_errors=False, show_sizes=False, |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 268 | show_detail=False, show_bloat=False, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 269 | list_error_boards=False, show_config=False): |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 270 | """Setup display options for the builder. |
| 271 | |
| 272 | show_errors: True to show summarised error/warning info |
| 273 | show_sizes: Show size deltas |
| 274 | show_detail: Show detail for each board |
| 275 | show_bloat: Show detail for each function |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 276 | list_error_boards: Show the boards which caused each error/warning |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 277 | show_config: Show config deltas |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 278 | """ |
| 279 | self._show_errors = show_errors |
| 280 | self._show_sizes = show_sizes |
| 281 | self._show_detail = show_detail |
| 282 | self._show_bloat = show_bloat |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 283 | self._list_error_boards = list_error_boards |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 284 | self._show_config = show_config |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 285 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 286 | def _AddTimestamp(self): |
| 287 | """Add a new timestamp to the list and record the build period. |
| 288 | |
| 289 | The build period is the length of time taken to perform a single |
| 290 | build (one board, one commit). |
| 291 | """ |
| 292 | now = datetime.now() |
| 293 | self._timestamps.append(now) |
| 294 | count = len(self._timestamps) |
| 295 | delta = self._timestamps[-1] - self._timestamps[0] |
| 296 | seconds = delta.total_seconds() |
| 297 | |
| 298 | # If we have enough data, estimate build period (time taken for a |
| 299 | # single build) and therefore completion time. |
| 300 | if count > 1 and self._next_delay_update < now: |
| 301 | self._next_delay_update = now + timedelta(seconds=2) |
| 302 | if seconds > 0: |
| 303 | self._build_period = float(seconds) / count |
| 304 | todo = self.count - self.upto |
| 305 | self._complete_delay = timedelta(microseconds= |
| 306 | self._build_period * todo * 1000000) |
| 307 | # Round it |
| 308 | self._complete_delay -= timedelta( |
| 309 | microseconds=self._complete_delay.microseconds) |
| 310 | |
| 311 | if seconds > 60: |
| 312 | self._timestamps.popleft() |
| 313 | count -= 1 |
| 314 | |
| 315 | def ClearLine(self, length): |
| 316 | """Clear any characters on the current line |
| 317 | |
| 318 | Make way for a new line of length 'length', by outputting enough |
| 319 | spaces to clear out the old line. Then remember the new length for |
| 320 | next time. |
| 321 | |
| 322 | Args: |
| 323 | length: Length of new line, in characters |
| 324 | """ |
| 325 | if length < self.last_line_len: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 326 | Print(' ' * (self.last_line_len - length), newline=False) |
| 327 | Print('\r', newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 328 | self.last_line_len = length |
| 329 | sys.stdout.flush() |
| 330 | |
| 331 | def SelectCommit(self, commit, checkout=True): |
| 332 | """Checkout the selected commit for this build |
| 333 | """ |
| 334 | self.commit = commit |
| 335 | if checkout and self.checkout: |
| 336 | gitutil.Checkout(commit.hash) |
| 337 | |
| 338 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): |
| 339 | """Run make |
| 340 | |
| 341 | Args: |
| 342 | commit: Commit object that is being built |
| 343 | brd: Board object that is being built |
Roger Meier | e0a0e55 | 2014-08-20 22:10:29 +0200 | [diff] [blame] | 344 | stage: Stage that we are at (mrproper, config, build) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 345 | cwd: Directory where make should be run |
| 346 | args: Arguments to pass to make |
| 347 | kwargs: Arguments to pass to command.RunPipe() |
| 348 | """ |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 349 | cmd = [self.gnu_make] + list(args) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 350 | result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
| 351 | cwd=cwd, raise_on_error=False, **kwargs) |
Simon Glass | 413f91a | 2015-02-05 22:06:12 -0700 | [diff] [blame] | 352 | if self.verbose_build: |
| 353 | result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout |
| 354 | result.combined = '%s\n' % (' '.join(cmd)) + result.combined |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 355 | return result |
| 356 | |
| 357 | def ProcessResult(self, result): |
| 358 | """Process the result of a build, showing progress information |
| 359 | |
| 360 | Args: |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 361 | result: A CommandResult object, which indicates the result for |
| 362 | a single build |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 363 | """ |
| 364 | col = terminal.Color() |
| 365 | if result: |
| 366 | target = result.brd.target |
| 367 | |
| 368 | if result.return_code < 0: |
| 369 | self.active = False |
| 370 | command.StopAll() |
| 371 | return |
| 372 | |
| 373 | self.upto += 1 |
| 374 | if result.return_code != 0: |
| 375 | self.fail += 1 |
| 376 | elif result.stderr: |
| 377 | self.warned += 1 |
| 378 | if result.already_done: |
| 379 | self.already_done += 1 |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 380 | if self._verbose: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 381 | Print('\r', newline=False) |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 382 | self.ClearLine(0) |
| 383 | boards_selected = {target : result.brd} |
| 384 | self.ResetResultSummary(boards_selected) |
| 385 | self.ProduceResultSummary(result.commit_upto, self.commits, |
| 386 | boards_selected) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 387 | else: |
| 388 | target = '(starting)' |
| 389 | |
| 390 | # Display separate counts for ok, warned and fail |
| 391 | ok = self.upto - self.warned - self.fail |
| 392 | line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) |
| 393 | line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) |
| 394 | line += self.col.Color(self.col.RED, '%5d' % self.fail) |
| 395 | |
| 396 | name = ' /%-5d ' % self.count |
| 397 | |
| 398 | # Add our current completion time estimate |
| 399 | self._AddTimestamp() |
| 400 | if self._complete_delay: |
| 401 | name += '%s : ' % self._complete_delay |
| 402 | # When building all boards for a commit, we can print a commit |
| 403 | # progress message. |
| 404 | if result and result.commit_upto is None: |
| 405 | name += 'commit %2d/%-3d' % (self.commit_upto + 1, |
| 406 | self.commit_count) |
| 407 | |
| 408 | name += target |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 409 | Print(line + name, newline=False) |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 410 | length = 14 + len(name) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 411 | self.ClearLine(length) |
| 412 | |
| 413 | def _GetOutputDir(self, commit_upto): |
| 414 | """Get the name of the output directory for a commit number |
| 415 | |
| 416 | The output directory is typically .../<branch>/<commit>. |
| 417 | |
| 418 | Args: |
| 419 | commit_upto: Commit number to use (0..self.count-1) |
| 420 | """ |
Simon Glass | e87bde1 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 421 | commit_dir = None |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 422 | if self.commits: |
| 423 | commit = self.commits[commit_upto] |
| 424 | subject = commit.subject.translate(trans_valid_chars) |
| 425 | commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, |
| 426 | self.commit_count, commit.hash, subject[:20])) |
Simon Glass | e87bde1 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 427 | elif not self.no_subdirs: |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 428 | commit_dir = 'current' |
Simon Glass | e87bde1 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 429 | if not commit_dir: |
| 430 | return self.base_dir |
| 431 | return os.path.join(self.base_dir, commit_dir) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 432 | |
| 433 | def GetBuildDir(self, commit_upto, target): |
| 434 | """Get the name of the build directory for a commit number |
| 435 | |
| 436 | The build directory is typically .../<branch>/<commit>/<target>. |
| 437 | |
| 438 | Args: |
| 439 | commit_upto: Commit number to use (0..self.count-1) |
| 440 | target: Target name |
| 441 | """ |
| 442 | output_dir = self._GetOutputDir(commit_upto) |
| 443 | return os.path.join(output_dir, target) |
| 444 | |
| 445 | def GetDoneFile(self, commit_upto, target): |
| 446 | """Get the name of the done file for a commit number |
| 447 | |
| 448 | Args: |
| 449 | commit_upto: Commit number to use (0..self.count-1) |
| 450 | target: Target name |
| 451 | """ |
| 452 | return os.path.join(self.GetBuildDir(commit_upto, target), 'done') |
| 453 | |
| 454 | def GetSizesFile(self, commit_upto, target): |
| 455 | """Get the name of the sizes file for a commit number |
| 456 | |
| 457 | Args: |
| 458 | commit_upto: Commit number to use (0..self.count-1) |
| 459 | target: Target name |
| 460 | """ |
| 461 | return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') |
| 462 | |
| 463 | def GetFuncSizesFile(self, commit_upto, target, elf_fname): |
| 464 | """Get the name of the funcsizes file for a commit number and ELF file |
| 465 | |
| 466 | Args: |
| 467 | commit_upto: Commit number to use (0..self.count-1) |
| 468 | target: Target name |
| 469 | elf_fname: Filename of elf image |
| 470 | """ |
| 471 | return os.path.join(self.GetBuildDir(commit_upto, target), |
| 472 | '%s.sizes' % elf_fname.replace('/', '-')) |
| 473 | |
| 474 | def GetObjdumpFile(self, commit_upto, target, elf_fname): |
| 475 | """Get the name of the objdump file for a commit number and ELF file |
| 476 | |
| 477 | Args: |
| 478 | commit_upto: Commit number to use (0..self.count-1) |
| 479 | target: Target name |
| 480 | elf_fname: Filename of elf image |
| 481 | """ |
| 482 | return os.path.join(self.GetBuildDir(commit_upto, target), |
| 483 | '%s.objdump' % elf_fname.replace('/', '-')) |
| 484 | |
| 485 | def GetErrFile(self, commit_upto, target): |
| 486 | """Get the name of the err file for a commit number |
| 487 | |
| 488 | Args: |
| 489 | commit_upto: Commit number to use (0..self.count-1) |
| 490 | target: Target name |
| 491 | """ |
| 492 | output_dir = self.GetBuildDir(commit_upto, target) |
| 493 | return os.path.join(output_dir, 'err') |
| 494 | |
| 495 | def FilterErrors(self, lines): |
| 496 | """Filter out errors in which we have no interest |
| 497 | |
| 498 | We should probably use map(). |
| 499 | |
| 500 | Args: |
| 501 | lines: List of error lines, each a string |
| 502 | Returns: |
| 503 | New list with only interesting lines included |
| 504 | """ |
| 505 | out_lines = [] |
| 506 | for line in lines: |
| 507 | if not self.re_make_err.search(line): |
| 508 | out_lines.append(line) |
| 509 | return out_lines |
| 510 | |
| 511 | def ReadFuncSizes(self, fname, fd): |
| 512 | """Read function sizes from the output of 'nm' |
| 513 | |
| 514 | Args: |
| 515 | fd: File containing data to read |
| 516 | fname: Filename we are reading from (just for errors) |
| 517 | |
| 518 | Returns: |
| 519 | Dictionary containing size of each function in bytes, indexed by |
| 520 | function name. |
| 521 | """ |
| 522 | sym = {} |
| 523 | for line in fd.readlines(): |
| 524 | try: |
| 525 | size, type, name = line[:-1].split() |
| 526 | except: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 527 | Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 528 | continue |
| 529 | if type in 'tTdDbB': |
| 530 | # function names begin with '.' on 64-bit powerpc |
| 531 | if '.' in name[1:]: |
| 532 | name = 'static.' + name.split('.')[0] |
| 533 | sym[name] = sym.get(name, 0) + int(size, 16) |
| 534 | return sym |
| 535 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 536 | def _ProcessConfig(self, fname): |
| 537 | """Read in a .config, autoconf.mk or autoconf.h file |
| 538 | |
| 539 | This function handles all config file types. It ignores comments and |
| 540 | any #defines which don't start with CONFIG_. |
| 541 | |
| 542 | Args: |
| 543 | fname: Filename to read |
| 544 | |
| 545 | Returns: |
| 546 | Dictionary: |
| 547 | key: Config name (e.g. CONFIG_DM) |
| 548 | value: Config value (e.g. 1) |
| 549 | """ |
| 550 | config = {} |
| 551 | if os.path.exists(fname): |
| 552 | with open(fname) as fd: |
| 553 | for line in fd: |
| 554 | line = line.strip() |
| 555 | if line.startswith('#define'): |
| 556 | values = line[8:].split(' ', 1) |
| 557 | if len(values) > 1: |
| 558 | key, value = values |
| 559 | else: |
| 560 | key = values[0] |
| 561 | value = '' |
| 562 | if not key.startswith('CONFIG_'): |
| 563 | continue |
| 564 | elif not line or line[0] in ['#', '*', '/']: |
| 565 | continue |
| 566 | else: |
| 567 | key, value = line.split('=', 1) |
| 568 | config[key] = value |
| 569 | return config |
| 570 | |
| 571 | def GetBuildOutcome(self, commit_upto, target, read_func_sizes, |
| 572 | read_config): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 573 | """Work out the outcome of a build. |
| 574 | |
| 575 | Args: |
| 576 | commit_upto: Commit number to check (0..n-1) |
| 577 | target: Target board to check |
| 578 | read_func_sizes: True to read function size information |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 579 | read_config: True to read .config and autoconf.h files |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 580 | |
| 581 | Returns: |
| 582 | Outcome object |
| 583 | """ |
| 584 | done_file = self.GetDoneFile(commit_upto, target) |
| 585 | sizes_file = self.GetSizesFile(commit_upto, target) |
| 586 | sizes = {} |
| 587 | func_sizes = {} |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 588 | config = {} |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 589 | if os.path.exists(done_file): |
| 590 | with open(done_file, 'r') as fd: |
| 591 | return_code = int(fd.readline()) |
| 592 | err_lines = [] |
| 593 | err_file = self.GetErrFile(commit_upto, target) |
| 594 | if os.path.exists(err_file): |
| 595 | with open(err_file, 'r') as fd: |
| 596 | err_lines = self.FilterErrors(fd.readlines()) |
| 597 | |
| 598 | # Decide whether the build was ok, failed or created warnings |
| 599 | if return_code: |
| 600 | rc = OUTCOME_ERROR |
| 601 | elif len(err_lines): |
| 602 | rc = OUTCOME_WARNING |
| 603 | else: |
| 604 | rc = OUTCOME_OK |
| 605 | |
| 606 | # Convert size information to our simple format |
| 607 | if os.path.exists(sizes_file): |
| 608 | with open(sizes_file, 'r') as fd: |
| 609 | for line in fd.readlines(): |
| 610 | values = line.split() |
| 611 | rodata = 0 |
| 612 | if len(values) > 6: |
| 613 | rodata = int(values[6], 16) |
| 614 | size_dict = { |
| 615 | 'all' : int(values[0]) + int(values[1]) + |
| 616 | int(values[2]), |
| 617 | 'text' : int(values[0]) - rodata, |
| 618 | 'data' : int(values[1]), |
| 619 | 'bss' : int(values[2]), |
| 620 | 'rodata' : rodata, |
| 621 | } |
| 622 | sizes[values[5]] = size_dict |
| 623 | |
| 624 | if read_func_sizes: |
| 625 | pattern = self.GetFuncSizesFile(commit_upto, target, '*') |
| 626 | for fname in glob.glob(pattern): |
| 627 | with open(fname, 'r') as fd: |
| 628 | dict_name = os.path.basename(fname).replace('.sizes', |
| 629 | '') |
| 630 | func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) |
| 631 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 632 | if read_config: |
| 633 | output_dir = self.GetBuildDir(commit_upto, target) |
| 634 | for name in CONFIG_FILENAMES: |
| 635 | fname = os.path.join(output_dir, name) |
| 636 | config[name] = self._ProcessConfig(fname) |
| 637 | |
| 638 | return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 639 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 640 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 641 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 642 | def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, |
| 643 | read_config): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 644 | """Calculate a summary of the results of building a commit. |
| 645 | |
| 646 | Args: |
| 647 | board_selected: Dict containing boards to summarise |
| 648 | commit_upto: Commit number to summarize (0..self.count-1) |
| 649 | read_func_sizes: True to read function size information |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 650 | read_config: True to read .config and autoconf.h files |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 651 | |
| 652 | Returns: |
| 653 | Tuple: |
| 654 | Dict containing boards which passed building this commit. |
| 655 | keyed by board.target |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 656 | List containing a summary of error lines |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 657 | Dict keyed by error line, containing a list of the Board |
| 658 | objects with that error |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 659 | List containing a summary of warning lines |
| 660 | Dict keyed by error line, containing a list of the Board |
| 661 | objects with that warning |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 662 | Dictionary keyed by filename - e.g. '.config'. Each |
| 663 | value is itself a dictionary: |
| 664 | key: config name |
| 665 | value: config value |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 666 | """ |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 667 | def AddLine(lines_summary, lines_boards, line, board): |
| 668 | line = line.rstrip() |
| 669 | if line in lines_boards: |
| 670 | lines_boards[line].append(board) |
| 671 | else: |
| 672 | lines_boards[line] = [board] |
| 673 | lines_summary.append(line) |
| 674 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 675 | board_dict = {} |
| 676 | err_lines_summary = [] |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 677 | err_lines_boards = {} |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 678 | warn_lines_summary = [] |
| 679 | warn_lines_boards = {} |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 680 | config = {} |
| 681 | for fname in CONFIG_FILENAMES: |
| 682 | config[fname] = {} |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 683 | |
| 684 | for board in boards_selected.itervalues(): |
| 685 | outcome = self.GetBuildOutcome(commit_upto, board.target, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 686 | read_func_sizes, read_config) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 687 | board_dict[board.target] = outcome |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 688 | last_func = None |
| 689 | last_was_warning = False |
| 690 | for line in outcome.err_lines: |
| 691 | if line: |
| 692 | if (self._re_function.match(line) or |
| 693 | self._re_files.match(line)): |
| 694 | last_func = line |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 695 | else: |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 696 | is_warning = self._re_warning.match(line) |
| 697 | is_note = self._re_note.match(line) |
| 698 | if is_warning or (last_was_warning and is_note): |
| 699 | if last_func: |
| 700 | AddLine(warn_lines_summary, warn_lines_boards, |
| 701 | last_func, board) |
| 702 | AddLine(warn_lines_summary, warn_lines_boards, |
| 703 | line, board) |
| 704 | else: |
| 705 | if last_func: |
| 706 | AddLine(err_lines_summary, err_lines_boards, |
| 707 | last_func, board) |
| 708 | AddLine(err_lines_summary, err_lines_boards, |
| 709 | line, board) |
| 710 | last_was_warning = is_warning |
| 711 | last_func = None |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 712 | for fname in CONFIG_FILENAMES: |
| 713 | config[fname] = {} |
| 714 | if outcome.config: |
| 715 | for key, value in outcome.config[fname].iteritems(): |
| 716 | config[fname][key] = value |
| 717 | |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 718 | return (board_dict, err_lines_summary, err_lines_boards, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 719 | warn_lines_summary, warn_lines_boards, config) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 720 | |
| 721 | def AddOutcome(self, board_dict, arch_list, changes, char, color): |
| 722 | """Add an output to our list of outcomes for each architecture |
| 723 | |
| 724 | This simple function adds failing boards (changes) to the |
| 725 | relevant architecture string, so we can print the results out |
| 726 | sorted by architecture. |
| 727 | |
| 728 | Args: |
| 729 | board_dict: Dict containing all boards |
| 730 | arch_list: Dict keyed by arch name. Value is a string containing |
| 731 | a list of board names which failed for that arch. |
| 732 | changes: List of boards to add to arch_list |
| 733 | color: terminal.Colour object |
| 734 | """ |
| 735 | done_arch = {} |
| 736 | for target in changes: |
| 737 | if target in board_dict: |
| 738 | arch = board_dict[target].arch |
| 739 | else: |
| 740 | arch = 'unknown' |
| 741 | str = self.col.Color(color, ' ' + target) |
| 742 | if not arch in done_arch: |
Simon Glass | 26f7237 | 2015-02-05 22:06:11 -0700 | [diff] [blame] | 743 | str = ' %s %s' % (self.col.Color(color, char), str) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 744 | done_arch[arch] = True |
| 745 | if not arch in arch_list: |
| 746 | arch_list[arch] = str |
| 747 | else: |
| 748 | arch_list[arch] += str |
| 749 | |
| 750 | |
| 751 | def ColourNum(self, num): |
| 752 | color = self.col.RED if num > 0 else self.col.GREEN |
| 753 | if num == 0: |
| 754 | return '0' |
| 755 | return self.col.Color(color, str(num)) |
| 756 | |
| 757 | def ResetResultSummary(self, board_selected): |
| 758 | """Reset the results summary ready for use. |
| 759 | |
| 760 | Set up the base board list to be all those selected, and set the |
| 761 | error lines to empty. |
| 762 | |
| 763 | Following this, calls to PrintResultSummary() will use this |
| 764 | information to work out what has changed. |
| 765 | |
| 766 | Args: |
| 767 | board_selected: Dict containing boards to summarise, keyed by |
| 768 | board.target |
| 769 | """ |
| 770 | self._base_board_dict = {} |
| 771 | for board in board_selected: |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 772 | self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 773 | self._base_err_lines = [] |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 774 | self._base_warn_lines = [] |
| 775 | self._base_err_line_boards = {} |
| 776 | self._base_warn_line_boards = {} |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 777 | self._base_config = {} |
| 778 | for fname in CONFIG_FILENAMES: |
| 779 | self._base_config[fname] = {} |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 780 | |
| 781 | def PrintFuncSizeDetail(self, fname, old, new): |
| 782 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 |
| 783 | delta, common = [], {} |
| 784 | |
| 785 | for a in old: |
| 786 | if a in new: |
| 787 | common[a] = 1 |
| 788 | |
| 789 | for name in old: |
| 790 | if name not in common: |
| 791 | remove += 1 |
| 792 | down += old[name] |
| 793 | delta.append([-old[name], name]) |
| 794 | |
| 795 | for name in new: |
| 796 | if name not in common: |
| 797 | add += 1 |
| 798 | up += new[name] |
| 799 | delta.append([new[name], name]) |
| 800 | |
| 801 | for name in common: |
| 802 | diff = new.get(name, 0) - old.get(name, 0) |
| 803 | if diff > 0: |
| 804 | grow, up = grow + 1, up + diff |
| 805 | elif diff < 0: |
| 806 | shrink, down = shrink + 1, down - diff |
| 807 | delta.append([diff, name]) |
| 808 | |
| 809 | delta.sort() |
| 810 | delta.reverse() |
| 811 | |
| 812 | args = [add, -remove, grow, -shrink, up, -down, up - down] |
| 813 | if max(args) == 0: |
| 814 | return |
| 815 | args = [self.ColourNum(x) for x in args] |
| 816 | indent = ' ' * 15 |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 817 | Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
| 818 | tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) |
| 819 | Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', |
| 820 | 'delta')) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 821 | for diff, name in delta: |
| 822 | if diff: |
| 823 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 824 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, |
| 825 | old.get(name, '-'), new.get(name,'-'), diff) |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 826 | Print(msg, colour=color) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 827 | |
| 828 | |
| 829 | def PrintSizeDetail(self, target_list, show_bloat): |
| 830 | """Show details size information for each board |
| 831 | |
| 832 | Args: |
| 833 | target_list: List of targets, each a dict containing: |
| 834 | 'target': Target name |
| 835 | 'total_diff': Total difference in bytes across all areas |
| 836 | <part_name>: Difference for that part |
| 837 | show_bloat: Show detail for each function |
| 838 | """ |
| 839 | targets_by_diff = sorted(target_list, reverse=True, |
| 840 | key=lambda x: x['_total_diff']) |
| 841 | for result in targets_by_diff: |
| 842 | printed_target = False |
| 843 | for name in sorted(result): |
| 844 | diff = result[name] |
| 845 | if name.startswith('_'): |
| 846 | continue |
| 847 | if diff != 0: |
| 848 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 849 | msg = ' %s %+d' % (name, diff) |
| 850 | if not printed_target: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 851 | Print('%10s %-15s:' % ('', result['_target']), |
| 852 | newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 853 | printed_target = True |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 854 | Print(msg, colour=color, newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 855 | if printed_target: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 856 | Print() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 857 | if show_bloat: |
| 858 | target = result['_target'] |
| 859 | outcome = result['_outcome'] |
| 860 | base_outcome = self._base_board_dict[target] |
| 861 | for fname in outcome.func_sizes: |
| 862 | self.PrintFuncSizeDetail(fname, |
| 863 | base_outcome.func_sizes[fname], |
| 864 | outcome.func_sizes[fname]) |
| 865 | |
| 866 | |
| 867 | def PrintSizeSummary(self, board_selected, board_dict, show_detail, |
| 868 | show_bloat): |
| 869 | """Print a summary of image sizes broken down by section. |
| 870 | |
| 871 | The summary takes the form of one line per architecture. The |
| 872 | line contains deltas for each of the sections (+ means the section |
| 873 | got bigger, - means smaller). The nunmbers are the average number |
| 874 | of bytes that a board in this section increased by. |
| 875 | |
| 876 | For example: |
| 877 | powerpc: (622 boards) text -0.0 |
| 878 | arm: (285 boards) text -0.0 |
| 879 | nds32: (3 boards) text -8.0 |
| 880 | |
| 881 | Args: |
| 882 | board_selected: Dict containing boards to summarise, keyed by |
| 883 | board.target |
| 884 | board_dict: Dict containing boards for which we built this |
| 885 | commit, keyed by board.target. The value is an Outcome object. |
| 886 | show_detail: Show detail for each board |
| 887 | show_bloat: Show detail for each function |
| 888 | """ |
| 889 | arch_list = {} |
| 890 | arch_count = {} |
| 891 | |
| 892 | # Calculate changes in size for different image parts |
| 893 | # The previous sizes are in Board.sizes, for each board |
| 894 | for target in board_dict: |
| 895 | if target not in board_selected: |
| 896 | continue |
| 897 | base_sizes = self._base_board_dict[target].sizes |
| 898 | outcome = board_dict[target] |
| 899 | sizes = outcome.sizes |
| 900 | |
| 901 | # Loop through the list of images, creating a dict of size |
| 902 | # changes for each image/part. We end up with something like |
| 903 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} |
| 904 | # which means that U-Boot data increased by 5 bytes and SPL |
| 905 | # text decreased by 4. |
| 906 | err = {'_target' : target} |
| 907 | for image in sizes: |
| 908 | if image in base_sizes: |
| 909 | base_image = base_sizes[image] |
| 910 | # Loop through the text, data, bss parts |
| 911 | for part in sorted(sizes[image]): |
| 912 | diff = sizes[image][part] - base_image[part] |
| 913 | col = None |
| 914 | if diff: |
| 915 | if image == 'u-boot': |
| 916 | name = part |
| 917 | else: |
| 918 | name = image + ':' + part |
| 919 | err[name] = diff |
| 920 | arch = board_selected[target].arch |
| 921 | if not arch in arch_count: |
| 922 | arch_count[arch] = 1 |
| 923 | else: |
| 924 | arch_count[arch] += 1 |
| 925 | if not sizes: |
| 926 | pass # Only add to our list when we have some stats |
| 927 | elif not arch in arch_list: |
| 928 | arch_list[arch] = [err] |
| 929 | else: |
| 930 | arch_list[arch].append(err) |
| 931 | |
| 932 | # We now have a list of image size changes sorted by arch |
| 933 | # Print out a summary of these |
| 934 | for arch, target_list in arch_list.iteritems(): |
| 935 | # Get total difference for each type |
| 936 | totals = {} |
| 937 | for result in target_list: |
| 938 | total = 0 |
| 939 | for name, diff in result.iteritems(): |
| 940 | if name.startswith('_'): |
| 941 | continue |
| 942 | total += diff |
| 943 | if name in totals: |
| 944 | totals[name] += diff |
| 945 | else: |
| 946 | totals[name] = diff |
| 947 | result['_total_diff'] = total |
| 948 | result['_outcome'] = board_dict[result['_target']] |
| 949 | |
| 950 | count = len(target_list) |
| 951 | printed_arch = False |
| 952 | for name in sorted(totals): |
| 953 | diff = totals[name] |
| 954 | if diff: |
| 955 | # Display the average difference in this name for this |
| 956 | # architecture |
| 957 | avg_diff = float(diff) / count |
| 958 | color = self.col.RED if avg_diff > 0 else self.col.GREEN |
| 959 | msg = ' %s %+1.1f' % (name, avg_diff) |
| 960 | if not printed_arch: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 961 | Print('%10s: (for %d/%d boards)' % (arch, count, |
| 962 | arch_count[arch]), newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 963 | printed_arch = True |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 964 | Print(msg, colour=color, newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 965 | |
| 966 | if printed_arch: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 967 | Print() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 968 | if show_detail: |
| 969 | self.PrintSizeDetail(target_list, show_bloat) |
| 970 | |
| 971 | |
| 972 | def PrintResultSummary(self, board_selected, board_dict, err_lines, |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 973 | err_line_boards, warn_lines, warn_line_boards, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 974 | config, show_sizes, show_detail, show_bloat, |
| 975 | show_config): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 976 | """Compare results with the base results and display delta. |
| 977 | |
| 978 | Only boards mentioned in board_selected will be considered. This |
| 979 | function is intended to be called repeatedly with the results of |
| 980 | each commit. It therefore shows a 'diff' between what it saw in |
| 981 | the last call and what it sees now. |
| 982 | |
| 983 | Args: |
| 984 | board_selected: Dict containing boards to summarise, keyed by |
| 985 | board.target |
| 986 | board_dict: Dict containing boards for which we built this |
| 987 | commit, keyed by board.target. The value is an Outcome object. |
| 988 | err_lines: A list of errors for this commit, or [] if there is |
| 989 | none, or we don't want to print errors |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 990 | err_line_boards: Dict keyed by error line, containing a list of |
| 991 | the Board objects with that error |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 992 | warn_lines: A list of warnings for this commit, or [] if there is |
| 993 | none, or we don't want to print errors |
| 994 | warn_line_boards: Dict keyed by warning line, containing a list of |
| 995 | the Board objects with that warning |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 996 | config: Dictionary keyed by filename - e.g. '.config'. Each |
| 997 | value is itself a dictionary: |
| 998 | key: config name |
| 999 | value: config value |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1000 | show_sizes: Show image size deltas |
| 1001 | show_detail: Show detail for each board |
| 1002 | show_bloat: Show detail for each function |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1003 | show_config: Show config changes |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1004 | """ |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1005 | def _BoardList(line, line_boards): |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1006 | """Helper function to get a line of boards containing a line |
| 1007 | |
| 1008 | Args: |
| 1009 | line: Error line to search for |
| 1010 | Return: |
| 1011 | String containing a list of boards with that error line, or |
| 1012 | '' if the user has not requested such a list |
| 1013 | """ |
| 1014 | if self._list_error_boards: |
| 1015 | names = [] |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1016 | for board in line_boards[line]: |
Simon Glass | 85620bb | 2014-10-16 01:05:55 -0600 | [diff] [blame] | 1017 | if not board.target in names: |
| 1018 | names.append(board.target) |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1019 | names_str = '(%s) ' % ','.join(names) |
| 1020 | else: |
| 1021 | names_str = '' |
| 1022 | return names_str |
| 1023 | |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1024 | def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, |
| 1025 | char): |
| 1026 | better_lines = [] |
| 1027 | worse_lines = [] |
| 1028 | for line in lines: |
| 1029 | if line not in base_lines: |
| 1030 | worse_lines.append(char + '+' + |
| 1031 | _BoardList(line, line_boards) + line) |
| 1032 | for line in base_lines: |
| 1033 | if line not in lines: |
| 1034 | better_lines.append(char + '-' + |
| 1035 | _BoardList(line, base_line_boards) + line) |
| 1036 | return better_lines, worse_lines |
| 1037 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1038 | def _CalcConfig(delta, name, config): |
| 1039 | """Calculate configuration changes |
| 1040 | |
| 1041 | Args: |
| 1042 | delta: Type of the delta, e.g. '+' |
| 1043 | name: name of the file which changed (e.g. .config) |
| 1044 | config: configuration change dictionary |
| 1045 | key: config name |
| 1046 | value: config value |
| 1047 | Returns: |
| 1048 | String containing the configuration changes which can be |
| 1049 | printed |
| 1050 | """ |
| 1051 | out = '' |
| 1052 | for key in sorted(config.keys()): |
| 1053 | out += '%s=%s ' % (key, config[key]) |
| 1054 | return '%5s %s: %s' % (delta, name, out) |
| 1055 | |
| 1056 | def _ShowConfig(name, config_plus, config_minus, config_change): |
| 1057 | """Show changes in configuration |
| 1058 | |
| 1059 | Args: |
| 1060 | config_plus: configurations added, dictionary |
| 1061 | key: config name |
| 1062 | value: config value |
| 1063 | config_minus: configurations removed, dictionary |
| 1064 | key: config name |
| 1065 | value: config value |
| 1066 | config_change: configurations changed, dictionary |
| 1067 | key: config name |
| 1068 | value: config value |
| 1069 | """ |
| 1070 | if config_plus: |
| 1071 | Print(_CalcConfig('+', name, config_plus), |
| 1072 | colour=self.col.GREEN) |
| 1073 | if config_minus: |
| 1074 | Print(_CalcConfig('-', name, config_minus), |
| 1075 | colour=self.col.RED) |
| 1076 | if config_change: |
| 1077 | Print(_CalcConfig('+/-', name, config_change), |
| 1078 | colour=self.col.YELLOW) |
| 1079 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1080 | better = [] # List of boards fixed since last commit |
| 1081 | worse = [] # List of new broken boards since last commit |
| 1082 | new = [] # List of boards that didn't exist last time |
| 1083 | unknown = [] # List of boards that were not built |
| 1084 | |
| 1085 | for target in board_dict: |
| 1086 | if target not in board_selected: |
| 1087 | continue |
| 1088 | |
| 1089 | # If the board was built last time, add its outcome to a list |
| 1090 | if target in self._base_board_dict: |
| 1091 | base_outcome = self._base_board_dict[target].rc |
| 1092 | outcome = board_dict[target] |
| 1093 | if outcome.rc == OUTCOME_UNKNOWN: |
| 1094 | unknown.append(target) |
| 1095 | elif outcome.rc < base_outcome: |
| 1096 | better.append(target) |
| 1097 | elif outcome.rc > base_outcome: |
| 1098 | worse.append(target) |
| 1099 | else: |
| 1100 | new.append(target) |
| 1101 | |
| 1102 | # Get a list of errors that have appeared, and disappeared |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1103 | better_err, worse_err = _CalcErrorDelta(self._base_err_lines, |
| 1104 | self._base_err_line_boards, err_lines, err_line_boards, '') |
| 1105 | better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, |
| 1106 | self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1107 | |
| 1108 | # Display results by arch |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1109 | if (better or worse or unknown or new or worse_err or better_err |
| 1110 | or worse_warn or better_warn): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1111 | arch_list = {} |
| 1112 | self.AddOutcome(board_selected, arch_list, better, '', |
| 1113 | self.col.GREEN) |
| 1114 | self.AddOutcome(board_selected, arch_list, worse, '+', |
| 1115 | self.col.RED) |
| 1116 | self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) |
| 1117 | if self._show_unknown: |
| 1118 | self.AddOutcome(board_selected, arch_list, unknown, '?', |
| 1119 | self.col.MAGENTA) |
| 1120 | for arch, target_list in arch_list.iteritems(): |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1121 | Print('%10s: %s' % (arch, target_list)) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1122 | self._error_lines += 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1123 | if better_err: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1124 | Print('\n'.join(better_err), colour=self.col.GREEN) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1125 | self._error_lines += 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1126 | if worse_err: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1127 | Print('\n'.join(worse_err), colour=self.col.RED) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1128 | self._error_lines += 1 |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1129 | if better_warn: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1130 | Print('\n'.join(better_warn), colour=self.col.CYAN) |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1131 | self._error_lines += 1 |
| 1132 | if worse_warn: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1133 | Print('\n'.join(worse_warn), colour=self.col.MAGENTA) |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1134 | self._error_lines += 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1135 | |
| 1136 | if show_sizes: |
| 1137 | self.PrintSizeSummary(board_selected, board_dict, show_detail, |
| 1138 | show_bloat) |
| 1139 | |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1140 | if show_config: |
| 1141 | all_config_plus = {} |
| 1142 | all_config_minus = {} |
| 1143 | all_config_change = {} |
| 1144 | for name in CONFIG_FILENAMES: |
| 1145 | if not config[name]: |
| 1146 | continue |
| 1147 | config_plus = {} |
| 1148 | config_minus = {} |
| 1149 | config_change = {} |
| 1150 | base = self._base_config[name] |
| 1151 | for key, value in config[name].iteritems(): |
| 1152 | if key not in base: |
| 1153 | config_plus[key] = value |
| 1154 | all_config_plus[key] = value |
| 1155 | for key, value in base.iteritems(): |
| 1156 | if key not in config[name]: |
| 1157 | config_minus[key] = value |
| 1158 | all_config_minus[key] = value |
| 1159 | for key, value in base.iteritems(): |
| 1160 | new_value = base[key] |
| 1161 | if key in config[name] and value != new_value: |
| 1162 | desc = '%s -> %s' % (value, new_value) |
| 1163 | config_change[key] = desc |
| 1164 | all_config_change[key] = desc |
| 1165 | _ShowConfig(name, config_plus, config_minus, config_change) |
| 1166 | _ShowConfig('all', all_config_plus, all_config_minus, |
| 1167 | all_config_change) |
| 1168 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1169 | # Save our updated information for the next call to this function |
| 1170 | self._base_board_dict = board_dict |
| 1171 | self._base_err_lines = err_lines |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1172 | self._base_warn_lines = warn_lines |
| 1173 | self._base_err_line_boards = err_line_boards |
| 1174 | self._base_warn_line_boards = warn_line_boards |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1175 | self._base_config = config |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1176 | |
| 1177 | # Get a list of boards that did not get built, if needed |
| 1178 | not_built = [] |
| 1179 | for board in board_selected: |
| 1180 | if not board in board_dict: |
| 1181 | not_built.append(board) |
| 1182 | if not_built: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1183 | Print("Boards not built (%d): %s" % (len(not_built), |
| 1184 | ', '.join(not_built))) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1185 | |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1186 | def ProduceResultSummary(self, commit_upto, commits, board_selected): |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1187 | (board_dict, err_lines, err_line_boards, warn_lines, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1188 | warn_line_boards, config) = self.GetResultSummary( |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1189 | board_selected, commit_upto, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1190 | read_func_sizes=self._show_bloat, |
| 1191 | read_config=self._show_config) |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1192 | if commits: |
| 1193 | msg = '%02d: %s' % (commit_upto + 1, |
| 1194 | commits[commit_upto].subject) |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1195 | Print(msg, colour=self.col.BLUE) |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1196 | self.PrintResultSummary(board_selected, board_dict, |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1197 | err_lines if self._show_errors else [], err_line_boards, |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1198 | warn_lines if self._show_errors else [], warn_line_boards, |
Simon Glass | db17fb8 | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1199 | config, self._show_sizes, self._show_detail, |
| 1200 | self._show_bloat, self._show_config) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1201 | |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1202 | def ShowSummary(self, commits, board_selected): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1203 | """Show a build summary for U-Boot for a given board list. |
| 1204 | |
| 1205 | Reset the result summary, then repeatedly call GetResultSummary on |
| 1206 | each commit's results, then display the differences we see. |
| 1207 | |
| 1208 | Args: |
| 1209 | commit: Commit objects to summarise |
| 1210 | board_selected: Dict containing boards to summarise |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1211 | """ |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1212 | self.commit_count = len(commits) if commits else 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1213 | self.commits = commits |
| 1214 | self.ResetResultSummary(board_selected) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1215 | self._error_lines = 0 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1216 | |
| 1217 | for commit_upto in range(0, self.commit_count, self._step): |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1218 | self.ProduceResultSummary(commit_upto, commits, board_selected) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1219 | if not self._error_lines: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1220 | Print('(no errors to report)', colour=self.col.GREEN) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1221 | |
| 1222 | |
| 1223 | def SetupBuild(self, board_selected, commits): |
| 1224 | """Set up ready to start a build. |
| 1225 | |
| 1226 | Args: |
| 1227 | board_selected: Selected boards to build |
| 1228 | commits: Selected commits to build |
| 1229 | """ |
| 1230 | # First work out how many commits we will build |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1231 | count = (self.commit_count + self._step - 1) / self._step |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1232 | self.count = len(board_selected) * count |
| 1233 | self.upto = self.warned = self.fail = 0 |
| 1234 | self._timestamps = collections.deque() |
| 1235 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1236 | def GetThreadDir(self, thread_num): |
| 1237 | """Get the directory path to the working dir for a thread. |
| 1238 | |
| 1239 | Args: |
| 1240 | thread_num: Number of thread to check. |
| 1241 | """ |
| 1242 | return os.path.join(self._working_dir, '%02d' % thread_num) |
| 1243 | |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1244 | def _PrepareThread(self, thread_num, setup_git): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1245 | """Prepare the working directory for a thread. |
| 1246 | |
| 1247 | This clones or fetches the repo into the thread's work directory. |
| 1248 | |
| 1249 | Args: |
| 1250 | thread_num: Thread number (0, 1, ...) |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1251 | setup_git: True to set up a git repo clone |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1252 | """ |
| 1253 | thread_dir = self.GetThreadDir(thread_num) |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1254 | builderthread.Mkdir(thread_dir) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1255 | git_dir = os.path.join(thread_dir, '.git') |
| 1256 | |
| 1257 | # Clone the repo if it doesn't already exist |
| 1258 | # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so |
| 1259 | # we have a private index but uses the origin repo's contents? |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1260 | if setup_git and self.git_dir: |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1261 | src_dir = os.path.abspath(self.git_dir) |
| 1262 | if os.path.exists(git_dir): |
| 1263 | gitutil.Fetch(git_dir, thread_dir) |
| 1264 | else: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1265 | Print('Cloning repo for thread %d' % thread_num) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1266 | gitutil.Clone(src_dir, thread_dir) |
| 1267 | |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1268 | def _PrepareWorkingSpace(self, max_threads, setup_git): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1269 | """Prepare the working directory for use. |
| 1270 | |
| 1271 | Set up the git repo for each thread. |
| 1272 | |
| 1273 | Args: |
| 1274 | max_threads: Maximum number of threads we expect to need. |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1275 | setup_git: True to set up a git repo clone |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1276 | """ |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1277 | builderthread.Mkdir(self._working_dir) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1278 | for thread in range(max_threads): |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1279 | self._PrepareThread(thread, setup_git) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1280 | |
| 1281 | def _PrepareOutputSpace(self): |
| 1282 | """Get the output directories ready to receive files. |
| 1283 | |
| 1284 | We delete any output directories which look like ones we need to |
| 1285 | create. Having left over directories is confusing when the user wants |
| 1286 | to check the output manually. |
| 1287 | """ |
Simon Glass | b9a9b7c | 2014-12-01 17:33:53 -0700 | [diff] [blame] | 1288 | if not self.commits: |
| 1289 | return |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1290 | dir_list = [] |
| 1291 | for commit_upto in range(self.commit_count): |
| 1292 | dir_list.append(self._GetOutputDir(commit_upto)) |
| 1293 | |
| 1294 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): |
| 1295 | if dirname not in dir_list: |
| 1296 | shutil.rmtree(dirname) |
| 1297 | |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1298 | def BuildBoards(self, commits, board_selected, keep_outputs, verbose): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1299 | """Build all commits for a list of boards |
| 1300 | |
| 1301 | Args: |
| 1302 | commits: List of commits to be build, each a Commit object |
| 1303 | boards_selected: Dict of selected boards, key is target name, |
| 1304 | value is Board object |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1305 | keep_outputs: True to save build output files |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1306 | verbose: Display build results as they are completed |
Simon Glass | c2f9107 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 1307 | Returns: |
| 1308 | Tuple containing: |
| 1309 | - number of boards that failed to build |
| 1310 | - number of boards that issued warnings |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1311 | """ |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1312 | self.commit_count = len(commits) if commits else 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1313 | self.commits = commits |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1314 | self._verbose = verbose |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1315 | |
| 1316 | self.ResetResultSummary(board_selected) |
Thierry Reding | 336d5bd | 2014-08-19 10:22:39 +0200 | [diff] [blame] | 1317 | builderthread.Mkdir(self.base_dir, parents = True) |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1318 | self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), |
| 1319 | commits is not None) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1320 | self._PrepareOutputSpace() |
| 1321 | self.SetupBuild(board_selected, commits) |
| 1322 | self.ProcessResult(None) |
| 1323 | |
| 1324 | # Create jobs to build all commits for each board |
| 1325 | for brd in board_selected.itervalues(): |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1326 | job = builderthread.BuilderJob() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1327 | job.board = brd |
| 1328 | job.commits = commits |
| 1329 | job.keep_outputs = keep_outputs |
| 1330 | job.step = self._step |
| 1331 | self.queue.put(job) |
| 1332 | |
| 1333 | # Wait until all jobs are started |
| 1334 | self.queue.join() |
| 1335 | |
| 1336 | # Wait until we have processed all output |
| 1337 | self.out_queue.join() |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1338 | Print() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1339 | self.ClearLine(0) |
Simon Glass | c2f9107 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 1340 | return (self.fail, self.warned) |