Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. |
| 2 | # |
| 3 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> |
| 4 | # |
Wolfgang Denk | d79de1d | 2013-07-08 09:37:19 +0200 | [diff] [blame] | 5 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 6 | # |
| 7 | |
| 8 | import collections |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 9 | from datetime import datetime, timedelta |
| 10 | import glob |
| 11 | import os |
| 12 | import re |
| 13 | import Queue |
| 14 | import shutil |
| 15 | import string |
| 16 | import sys |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 17 | import time |
| 18 | |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 19 | import builderthread |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 20 | import command |
| 21 | import gitutil |
| 22 | import terminal |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 23 | from terminal import Print |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 24 | import toolchain |
| 25 | |
| 26 | |
| 27 | """ |
| 28 | Theory of Operation |
| 29 | |
| 30 | Please see README for user documentation, and you should be familiar with |
| 31 | that before trying to make sense of this. |
| 32 | |
| 33 | Buildman works by keeping the machine as busy as possible, building different |
| 34 | commits for different boards on multiple CPUs at once. |
| 35 | |
| 36 | The source repo (self.git_dir) contains all the commits to be built. Each |
| 37 | thread works on a single board at a time. It checks out the first commit, |
| 38 | configures it for that board, then builds it. Then it checks out the next |
| 39 | commit and builds it (typically without re-configuring). When it runs out |
| 40 | of commits, it gets another job from the builder and starts again with that |
| 41 | board. |
| 42 | |
| 43 | Clearly the builder threads could work either way - they could check out a |
| 44 | commit and then built it for all boards. Using separate directories for each |
| 45 | commit/board pair they could leave their build product around afterwards |
| 46 | also. |
| 47 | |
| 48 | The intent behind building a single board for multiple commits, is to make |
| 49 | use of incremental builds. Since each commit is built incrementally from |
| 50 | the previous one, builds are faster. Reconfiguring for a different board |
| 51 | removes all intermediate object files. |
| 52 | |
| 53 | Many threads can be working at once, but each has its own working directory. |
| 54 | When a thread finishes a build, it puts the output files into a result |
| 55 | directory. |
| 56 | |
| 57 | The base directory used by buildman is normally '../<branch>', i.e. |
| 58 | a directory higher than the source repository and named after the branch |
| 59 | being built. |
| 60 | |
| 61 | Within the base directory, we have one subdirectory for each commit. Within |
| 62 | that is one subdirectory for each board. Within that is the build output for |
| 63 | that commit/board combination. |
| 64 | |
| 65 | Buildman also create working directories for each thread, in a .bm-work/ |
| 66 | subdirectory in the base dir. |
| 67 | |
| 68 | As an example, say we are building branch 'us-net' for boards 'sandbox' and |
| 69 | 'seaboard', and say that us-net has two commits. We will have directories |
| 70 | like this: |
| 71 | |
| 72 | us-net/ base directory |
| 73 | 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ |
| 74 | sandbox/ |
| 75 | u-boot.bin |
| 76 | seaboard/ |
| 77 | u-boot.bin |
| 78 | 02_of_02_g4ed4ebc_net--Check-tftp-comp/ |
| 79 | sandbox/ |
| 80 | u-boot.bin |
| 81 | seaboard/ |
| 82 | u-boot.bin |
| 83 | .bm-work/ |
| 84 | 00/ working directory for thread 0 (contains source checkout) |
| 85 | build/ build output |
| 86 | 01/ working directory for thread 1 |
| 87 | build/ build output |
| 88 | ... |
| 89 | u-boot/ source directory |
| 90 | .git/ repository |
| 91 | """ |
| 92 | |
| 93 | # Possible build outcomes |
| 94 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) |
| 95 | |
| 96 | # Translate a commit subject into a valid filename |
| 97 | trans_valid_chars = string.maketrans("/: ", "---") |
| 98 | |
| 99 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 100 | class Builder: |
| 101 | """Class for building U-Boot for a particular commit. |
| 102 | |
| 103 | Public members: (many should ->private) |
| 104 | active: True if the builder is active and has not been stopped |
| 105 | already_done: Number of builds already completed |
| 106 | base_dir: Base directory to use for builder |
| 107 | checkout: True to check out source, False to skip that step. |
| 108 | This is used for testing. |
| 109 | col: terminal.Color() object |
| 110 | count: Number of commits to build |
| 111 | do_make: Method to call to invoke Make |
| 112 | fail: Number of builds that failed due to error |
| 113 | force_build: Force building even if a build already exists |
| 114 | force_config_on_failure: If a commit fails for a board, disable |
| 115 | incremental building for the next commit we build for that |
| 116 | board, so that we will see all warnings/errors again. |
Simon Glass | 7041c39 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 117 | force_build_failures: If a previously-built build (i.e. built on |
| 118 | a previous run of buildman) is marked as failed, rebuild it. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 119 | git_dir: Git directory containing source repository |
| 120 | last_line_len: Length of the last line we printed (used for erasing |
| 121 | it with new progress information) |
| 122 | num_jobs: Number of jobs to run at once (passed to make as -j) |
| 123 | num_threads: Number of builder threads to run |
| 124 | out_queue: Queue of results to process |
| 125 | re_make_err: Compiled regular expression for ignore_lines |
| 126 | queue: Queue of jobs to run |
| 127 | threads: List of active threads |
| 128 | toolchains: Toolchains object to use for building |
| 129 | upto: Current commit number we are building (0.count-1) |
| 130 | warned: Number of builds that produced at least one warning |
Simon Glass | f3018b7a | 2014-07-14 17:51:02 -0600 | [diff] [blame] | 131 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
| 132 | incremental building, where buildman reconfigures on the first |
| 133 | commit for a baord, and then just does an incremental build for |
| 134 | the following commits. In fact buildman will reconfigure and |
| 135 | retry for any failing commits, so generally the only effect of |
| 136 | this option is to slow things down. |
Simon Glass | 38df2e2 | 2014-07-14 17:51:03 -0600 | [diff] [blame] | 137 | in_tree: Build U-Boot in-tree instead of specifying an output |
| 138 | directory separate from the source code. This option is really |
| 139 | only useful for testing in-tree builds. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 140 | |
| 141 | Private members: |
| 142 | _base_board_dict: Last-summarised Dict of boards |
| 143 | _base_err_lines: Last-summarised list of errors |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 144 | _base_warn_lines: Last-summarised list of warnings |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 145 | _build_period_us: Time taken for a single build (float object). |
| 146 | _complete_delay: Expected delay until completion (timedelta) |
| 147 | _next_delay_update: Next time we plan to display a progress update |
| 148 | (datatime) |
| 149 | _show_unknown: Show unknown boards (those not built) in summary |
| 150 | _timestamps: List of timestamps for the completion of the last |
| 151 | last _timestamp_count builds. Each is a datetime object. |
| 152 | _timestamp_count: Number of timestamps to keep in our list. |
| 153 | _working_dir: Base working directory containing all threads |
| 154 | """ |
| 155 | class Outcome: |
| 156 | """Records a build outcome for a single make invocation |
| 157 | |
| 158 | Public Members: |
| 159 | rc: Outcome value (OUTCOME_...) |
| 160 | err_lines: List of error lines or [] if none |
| 161 | sizes: Dictionary of image size information, keyed by filename |
| 162 | - Each value is itself a dictionary containing |
| 163 | values for 'text', 'data' and 'bss', being the integer |
| 164 | size in bytes of each section. |
| 165 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each |
| 166 | value is itself a dictionary: |
| 167 | key: function name |
| 168 | value: Size of function in bytes |
| 169 | """ |
| 170 | def __init__(self, rc, err_lines, sizes, func_sizes): |
| 171 | self.rc = rc |
| 172 | self.err_lines = err_lines |
| 173 | self.sizes = sizes |
| 174 | self.func_sizes = func_sizes |
| 175 | |
| 176 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 177 | gnu_make='make', checkout=True, show_unknown=True, step=1): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 178 | """Create a new Builder object |
| 179 | |
| 180 | Args: |
| 181 | toolchains: Toolchains object to use for building |
| 182 | base_dir: Base directory to use for builder |
| 183 | git_dir: Git directory containing source repository |
| 184 | num_threads: Number of builder threads to run |
| 185 | num_jobs: Number of jobs to run at once (passed to make as -j) |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 186 | gnu_make: the command name of GNU Make. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 187 | checkout: True to check out source, False to skip that step. |
| 188 | This is used for testing. |
| 189 | show_unknown: Show unknown boards (those not built) in summary |
| 190 | step: 1 to process every commit, n to process every nth commit |
| 191 | """ |
| 192 | self.toolchains = toolchains |
| 193 | self.base_dir = base_dir |
| 194 | self._working_dir = os.path.join(base_dir, '.bm-work') |
| 195 | self.threads = [] |
| 196 | self.active = True |
| 197 | self.do_make = self.Make |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 198 | self.gnu_make = gnu_make |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 199 | self.checkout = checkout |
| 200 | self.num_threads = num_threads |
| 201 | self.num_jobs = num_jobs |
| 202 | self.already_done = 0 |
| 203 | self.force_build = False |
| 204 | self.git_dir = git_dir |
| 205 | self._show_unknown = show_unknown |
| 206 | self._timestamp_count = 10 |
| 207 | self._build_period_us = None |
| 208 | self._complete_delay = None |
| 209 | self._next_delay_update = datetime.now() |
| 210 | self.force_config_on_failure = True |
Simon Glass | 7041c39 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 211 | self.force_build_failures = False |
Simon Glass | f3018b7a | 2014-07-14 17:51:02 -0600 | [diff] [blame] | 212 | self.force_reconfig = False |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 213 | self._step = step |
Simon Glass | 38df2e2 | 2014-07-14 17:51:03 -0600 | [diff] [blame] | 214 | self.in_tree = False |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 215 | self._error_lines = 0 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 216 | |
| 217 | self.col = terminal.Color() |
| 218 | |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 219 | self._re_function = re.compile('(.*): In function.*') |
| 220 | self._re_files = re.compile('In file included from.*') |
| 221 | self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') |
| 222 | self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') |
| 223 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 224 | self.queue = Queue.Queue() |
| 225 | self.out_queue = Queue.Queue() |
| 226 | for i in range(self.num_threads): |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 227 | t = builderthread.BuilderThread(self, i) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 228 | t.setDaemon(True) |
| 229 | t.start() |
| 230 | self.threads.append(t) |
| 231 | |
| 232 | self.last_line_len = 0 |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 233 | t = builderthread.ResultThread(self) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 234 | t.setDaemon(True) |
| 235 | t.start() |
| 236 | self.threads.append(t) |
| 237 | |
| 238 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] |
| 239 | self.re_make_err = re.compile('|'.join(ignore_lines)) |
| 240 | |
| 241 | def __del__(self): |
| 242 | """Get rid of all threads created by the builder""" |
| 243 | for t in self.threads: |
| 244 | del t |
| 245 | |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 246 | def SetDisplayOptions(self, show_errors=False, show_sizes=False, |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 247 | show_detail=False, show_bloat=False, |
| 248 | list_error_boards=False): |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 249 | """Setup display options for the builder. |
| 250 | |
| 251 | show_errors: True to show summarised error/warning info |
| 252 | show_sizes: Show size deltas |
| 253 | show_detail: Show detail for each board |
| 254 | show_bloat: Show detail for each function |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 255 | list_error_boards: Show the boards which caused each error/warning |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 256 | """ |
| 257 | self._show_errors = show_errors |
| 258 | self._show_sizes = show_sizes |
| 259 | self._show_detail = show_detail |
| 260 | self._show_bloat = show_bloat |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 261 | self._list_error_boards = list_error_boards |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 262 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 263 | def _AddTimestamp(self): |
| 264 | """Add a new timestamp to the list and record the build period. |
| 265 | |
| 266 | The build period is the length of time taken to perform a single |
| 267 | build (one board, one commit). |
| 268 | """ |
| 269 | now = datetime.now() |
| 270 | self._timestamps.append(now) |
| 271 | count = len(self._timestamps) |
| 272 | delta = self._timestamps[-1] - self._timestamps[0] |
| 273 | seconds = delta.total_seconds() |
| 274 | |
| 275 | # If we have enough data, estimate build period (time taken for a |
| 276 | # single build) and therefore completion time. |
| 277 | if count > 1 and self._next_delay_update < now: |
| 278 | self._next_delay_update = now + timedelta(seconds=2) |
| 279 | if seconds > 0: |
| 280 | self._build_period = float(seconds) / count |
| 281 | todo = self.count - self.upto |
| 282 | self._complete_delay = timedelta(microseconds= |
| 283 | self._build_period * todo * 1000000) |
| 284 | # Round it |
| 285 | self._complete_delay -= timedelta( |
| 286 | microseconds=self._complete_delay.microseconds) |
| 287 | |
| 288 | if seconds > 60: |
| 289 | self._timestamps.popleft() |
| 290 | count -= 1 |
| 291 | |
| 292 | def ClearLine(self, length): |
| 293 | """Clear any characters on the current line |
| 294 | |
| 295 | Make way for a new line of length 'length', by outputting enough |
| 296 | spaces to clear out the old line. Then remember the new length for |
| 297 | next time. |
| 298 | |
| 299 | Args: |
| 300 | length: Length of new line, in characters |
| 301 | """ |
| 302 | if length < self.last_line_len: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 303 | Print(' ' * (self.last_line_len - length), newline=False) |
| 304 | Print('\r', newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 305 | self.last_line_len = length |
| 306 | sys.stdout.flush() |
| 307 | |
| 308 | def SelectCommit(self, commit, checkout=True): |
| 309 | """Checkout the selected commit for this build |
| 310 | """ |
| 311 | self.commit = commit |
| 312 | if checkout and self.checkout: |
| 313 | gitutil.Checkout(commit.hash) |
| 314 | |
| 315 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): |
| 316 | """Run make |
| 317 | |
| 318 | Args: |
| 319 | commit: Commit object that is being built |
| 320 | brd: Board object that is being built |
Roger Meier | e0a0e55 | 2014-08-20 22:10:29 +0200 | [diff] [blame] | 321 | stage: Stage that we are at (mrproper, config, build) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 322 | cwd: Directory where make should be run |
| 323 | args: Arguments to pass to make |
| 324 | kwargs: Arguments to pass to command.RunPipe() |
| 325 | """ |
Masahiro Yamada | 1fe610d | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 326 | cmd = [self.gnu_make] + list(args) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 327 | result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
| 328 | cwd=cwd, raise_on_error=False, **kwargs) |
| 329 | return result |
| 330 | |
| 331 | def ProcessResult(self, result): |
| 332 | """Process the result of a build, showing progress information |
| 333 | |
| 334 | Args: |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 335 | result: A CommandResult object, which indicates the result for |
| 336 | a single build |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 337 | """ |
| 338 | col = terminal.Color() |
| 339 | if result: |
| 340 | target = result.brd.target |
| 341 | |
| 342 | if result.return_code < 0: |
| 343 | self.active = False |
| 344 | command.StopAll() |
| 345 | return |
| 346 | |
| 347 | self.upto += 1 |
| 348 | if result.return_code != 0: |
| 349 | self.fail += 1 |
| 350 | elif result.stderr: |
| 351 | self.warned += 1 |
| 352 | if result.already_done: |
| 353 | self.already_done += 1 |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 354 | if self._verbose: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 355 | Print('\r', newline=False) |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 356 | self.ClearLine(0) |
| 357 | boards_selected = {target : result.brd} |
| 358 | self.ResetResultSummary(boards_selected) |
| 359 | self.ProduceResultSummary(result.commit_upto, self.commits, |
| 360 | boards_selected) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 361 | else: |
| 362 | target = '(starting)' |
| 363 | |
| 364 | # Display separate counts for ok, warned and fail |
| 365 | ok = self.upto - self.warned - self.fail |
| 366 | line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) |
| 367 | line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) |
| 368 | line += self.col.Color(self.col.RED, '%5d' % self.fail) |
| 369 | |
| 370 | name = ' /%-5d ' % self.count |
| 371 | |
| 372 | # Add our current completion time estimate |
| 373 | self._AddTimestamp() |
| 374 | if self._complete_delay: |
| 375 | name += '%s : ' % self._complete_delay |
| 376 | # When building all boards for a commit, we can print a commit |
| 377 | # progress message. |
| 378 | if result and result.commit_upto is None: |
| 379 | name += 'commit %2d/%-3d' % (self.commit_upto + 1, |
| 380 | self.commit_count) |
| 381 | |
| 382 | name += target |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 383 | Print(line + name, newline=False) |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 384 | length = 14 + len(name) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 385 | self.ClearLine(length) |
| 386 | |
| 387 | def _GetOutputDir(self, commit_upto): |
| 388 | """Get the name of the output directory for a commit number |
| 389 | |
| 390 | The output directory is typically .../<branch>/<commit>. |
| 391 | |
| 392 | Args: |
| 393 | commit_upto: Commit number to use (0..self.count-1) |
| 394 | """ |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 395 | if self.commits: |
| 396 | commit = self.commits[commit_upto] |
| 397 | subject = commit.subject.translate(trans_valid_chars) |
| 398 | commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, |
| 399 | self.commit_count, commit.hash, subject[:20])) |
| 400 | else: |
| 401 | commit_dir = 'current' |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 402 | output_dir = os.path.join(self.base_dir, commit_dir) |
| 403 | return output_dir |
| 404 | |
| 405 | def GetBuildDir(self, commit_upto, target): |
| 406 | """Get the name of the build directory for a commit number |
| 407 | |
| 408 | The build directory is typically .../<branch>/<commit>/<target>. |
| 409 | |
| 410 | Args: |
| 411 | commit_upto: Commit number to use (0..self.count-1) |
| 412 | target: Target name |
| 413 | """ |
| 414 | output_dir = self._GetOutputDir(commit_upto) |
| 415 | return os.path.join(output_dir, target) |
| 416 | |
| 417 | def GetDoneFile(self, commit_upto, target): |
| 418 | """Get the name of the done file for a commit number |
| 419 | |
| 420 | Args: |
| 421 | commit_upto: Commit number to use (0..self.count-1) |
| 422 | target: Target name |
| 423 | """ |
| 424 | return os.path.join(self.GetBuildDir(commit_upto, target), 'done') |
| 425 | |
| 426 | def GetSizesFile(self, commit_upto, target): |
| 427 | """Get the name of the sizes file for a commit number |
| 428 | |
| 429 | Args: |
| 430 | commit_upto: Commit number to use (0..self.count-1) |
| 431 | target: Target name |
| 432 | """ |
| 433 | return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') |
| 434 | |
| 435 | def GetFuncSizesFile(self, commit_upto, target, elf_fname): |
| 436 | """Get the name of the funcsizes file for a commit number and ELF file |
| 437 | |
| 438 | Args: |
| 439 | commit_upto: Commit number to use (0..self.count-1) |
| 440 | target: Target name |
| 441 | elf_fname: Filename of elf image |
| 442 | """ |
| 443 | return os.path.join(self.GetBuildDir(commit_upto, target), |
| 444 | '%s.sizes' % elf_fname.replace('/', '-')) |
| 445 | |
| 446 | def GetObjdumpFile(self, commit_upto, target, elf_fname): |
| 447 | """Get the name of the objdump file for a commit number and ELF file |
| 448 | |
| 449 | Args: |
| 450 | commit_upto: Commit number to use (0..self.count-1) |
| 451 | target: Target name |
| 452 | elf_fname: Filename of elf image |
| 453 | """ |
| 454 | return os.path.join(self.GetBuildDir(commit_upto, target), |
| 455 | '%s.objdump' % elf_fname.replace('/', '-')) |
| 456 | |
| 457 | def GetErrFile(self, commit_upto, target): |
| 458 | """Get the name of the err file for a commit number |
| 459 | |
| 460 | Args: |
| 461 | commit_upto: Commit number to use (0..self.count-1) |
| 462 | target: Target name |
| 463 | """ |
| 464 | output_dir = self.GetBuildDir(commit_upto, target) |
| 465 | return os.path.join(output_dir, 'err') |
| 466 | |
| 467 | def FilterErrors(self, lines): |
| 468 | """Filter out errors in which we have no interest |
| 469 | |
| 470 | We should probably use map(). |
| 471 | |
| 472 | Args: |
| 473 | lines: List of error lines, each a string |
| 474 | Returns: |
| 475 | New list with only interesting lines included |
| 476 | """ |
| 477 | out_lines = [] |
| 478 | for line in lines: |
| 479 | if not self.re_make_err.search(line): |
| 480 | out_lines.append(line) |
| 481 | return out_lines |
| 482 | |
| 483 | def ReadFuncSizes(self, fname, fd): |
| 484 | """Read function sizes from the output of 'nm' |
| 485 | |
| 486 | Args: |
| 487 | fd: File containing data to read |
| 488 | fname: Filename we are reading from (just for errors) |
| 489 | |
| 490 | Returns: |
| 491 | Dictionary containing size of each function in bytes, indexed by |
| 492 | function name. |
| 493 | """ |
| 494 | sym = {} |
| 495 | for line in fd.readlines(): |
| 496 | try: |
| 497 | size, type, name = line[:-1].split() |
| 498 | except: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 499 | Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 500 | continue |
| 501 | if type in 'tTdDbB': |
| 502 | # function names begin with '.' on 64-bit powerpc |
| 503 | if '.' in name[1:]: |
| 504 | name = 'static.' + name.split('.')[0] |
| 505 | sym[name] = sym.get(name, 0) + int(size, 16) |
| 506 | return sym |
| 507 | |
| 508 | def GetBuildOutcome(self, commit_upto, target, read_func_sizes): |
| 509 | """Work out the outcome of a build. |
| 510 | |
| 511 | Args: |
| 512 | commit_upto: Commit number to check (0..n-1) |
| 513 | target: Target board to check |
| 514 | read_func_sizes: True to read function size information |
| 515 | |
| 516 | Returns: |
| 517 | Outcome object |
| 518 | """ |
| 519 | done_file = self.GetDoneFile(commit_upto, target) |
| 520 | sizes_file = self.GetSizesFile(commit_upto, target) |
| 521 | sizes = {} |
| 522 | func_sizes = {} |
| 523 | if os.path.exists(done_file): |
| 524 | with open(done_file, 'r') as fd: |
| 525 | return_code = int(fd.readline()) |
| 526 | err_lines = [] |
| 527 | err_file = self.GetErrFile(commit_upto, target) |
| 528 | if os.path.exists(err_file): |
| 529 | with open(err_file, 'r') as fd: |
| 530 | err_lines = self.FilterErrors(fd.readlines()) |
| 531 | |
| 532 | # Decide whether the build was ok, failed or created warnings |
| 533 | if return_code: |
| 534 | rc = OUTCOME_ERROR |
| 535 | elif len(err_lines): |
| 536 | rc = OUTCOME_WARNING |
| 537 | else: |
| 538 | rc = OUTCOME_OK |
| 539 | |
| 540 | # Convert size information to our simple format |
| 541 | if os.path.exists(sizes_file): |
| 542 | with open(sizes_file, 'r') as fd: |
| 543 | for line in fd.readlines(): |
| 544 | values = line.split() |
| 545 | rodata = 0 |
| 546 | if len(values) > 6: |
| 547 | rodata = int(values[6], 16) |
| 548 | size_dict = { |
| 549 | 'all' : int(values[0]) + int(values[1]) + |
| 550 | int(values[2]), |
| 551 | 'text' : int(values[0]) - rodata, |
| 552 | 'data' : int(values[1]), |
| 553 | 'bss' : int(values[2]), |
| 554 | 'rodata' : rodata, |
| 555 | } |
| 556 | sizes[values[5]] = size_dict |
| 557 | |
| 558 | if read_func_sizes: |
| 559 | pattern = self.GetFuncSizesFile(commit_upto, target, '*') |
| 560 | for fname in glob.glob(pattern): |
| 561 | with open(fname, 'r') as fd: |
| 562 | dict_name = os.path.basename(fname).replace('.sizes', |
| 563 | '') |
| 564 | func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) |
| 565 | |
| 566 | return Builder.Outcome(rc, err_lines, sizes, func_sizes) |
| 567 | |
| 568 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) |
| 569 | |
| 570 | def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): |
| 571 | """Calculate a summary of the results of building a commit. |
| 572 | |
| 573 | Args: |
| 574 | board_selected: Dict containing boards to summarise |
| 575 | commit_upto: Commit number to summarize (0..self.count-1) |
| 576 | read_func_sizes: True to read function size information |
| 577 | |
| 578 | Returns: |
| 579 | Tuple: |
| 580 | Dict containing boards which passed building this commit. |
| 581 | keyed by board.target |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 582 | List containing a summary of error lines |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 583 | Dict keyed by error line, containing a list of the Board |
| 584 | objects with that error |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 585 | List containing a summary of warning lines |
| 586 | Dict keyed by error line, containing a list of the Board |
| 587 | objects with that warning |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 588 | """ |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 589 | def AddLine(lines_summary, lines_boards, line, board): |
| 590 | line = line.rstrip() |
| 591 | if line in lines_boards: |
| 592 | lines_boards[line].append(board) |
| 593 | else: |
| 594 | lines_boards[line] = [board] |
| 595 | lines_summary.append(line) |
| 596 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 597 | board_dict = {} |
| 598 | err_lines_summary = [] |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 599 | err_lines_boards = {} |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 600 | warn_lines_summary = [] |
| 601 | warn_lines_boards = {} |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 602 | |
| 603 | for board in boards_selected.itervalues(): |
| 604 | outcome = self.GetBuildOutcome(commit_upto, board.target, |
| 605 | read_func_sizes) |
| 606 | board_dict[board.target] = outcome |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 607 | last_func = None |
| 608 | last_was_warning = False |
| 609 | for line in outcome.err_lines: |
| 610 | if line: |
| 611 | if (self._re_function.match(line) or |
| 612 | self._re_files.match(line)): |
| 613 | last_func = line |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 614 | else: |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 615 | is_warning = self._re_warning.match(line) |
| 616 | is_note = self._re_note.match(line) |
| 617 | if is_warning or (last_was_warning and is_note): |
| 618 | if last_func: |
| 619 | AddLine(warn_lines_summary, warn_lines_boards, |
| 620 | last_func, board) |
| 621 | AddLine(warn_lines_summary, warn_lines_boards, |
| 622 | line, board) |
| 623 | else: |
| 624 | if last_func: |
| 625 | AddLine(err_lines_summary, err_lines_boards, |
| 626 | last_func, board) |
| 627 | AddLine(err_lines_summary, err_lines_boards, |
| 628 | line, board) |
| 629 | last_was_warning = is_warning |
| 630 | last_func = None |
| 631 | return (board_dict, err_lines_summary, err_lines_boards, |
| 632 | warn_lines_summary, warn_lines_boards) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 633 | |
| 634 | def AddOutcome(self, board_dict, arch_list, changes, char, color): |
| 635 | """Add an output to our list of outcomes for each architecture |
| 636 | |
| 637 | This simple function adds failing boards (changes) to the |
| 638 | relevant architecture string, so we can print the results out |
| 639 | sorted by architecture. |
| 640 | |
| 641 | Args: |
| 642 | board_dict: Dict containing all boards |
| 643 | arch_list: Dict keyed by arch name. Value is a string containing |
| 644 | a list of board names which failed for that arch. |
| 645 | changes: List of boards to add to arch_list |
| 646 | color: terminal.Colour object |
| 647 | """ |
| 648 | done_arch = {} |
| 649 | for target in changes: |
| 650 | if target in board_dict: |
| 651 | arch = board_dict[target].arch |
| 652 | else: |
| 653 | arch = 'unknown' |
| 654 | str = self.col.Color(color, ' ' + target) |
| 655 | if not arch in done_arch: |
| 656 | str = self.col.Color(color, char) + ' ' + str |
| 657 | done_arch[arch] = True |
| 658 | if not arch in arch_list: |
| 659 | arch_list[arch] = str |
| 660 | else: |
| 661 | arch_list[arch] += str |
| 662 | |
| 663 | |
| 664 | def ColourNum(self, num): |
| 665 | color = self.col.RED if num > 0 else self.col.GREEN |
| 666 | if num == 0: |
| 667 | return '0' |
| 668 | return self.col.Color(color, str(num)) |
| 669 | |
| 670 | def ResetResultSummary(self, board_selected): |
| 671 | """Reset the results summary ready for use. |
| 672 | |
| 673 | Set up the base board list to be all those selected, and set the |
| 674 | error lines to empty. |
| 675 | |
| 676 | Following this, calls to PrintResultSummary() will use this |
| 677 | information to work out what has changed. |
| 678 | |
| 679 | Args: |
| 680 | board_selected: Dict containing boards to summarise, keyed by |
| 681 | board.target |
| 682 | """ |
| 683 | self._base_board_dict = {} |
| 684 | for board in board_selected: |
| 685 | self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) |
| 686 | self._base_err_lines = [] |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 687 | self._base_warn_lines = [] |
| 688 | self._base_err_line_boards = {} |
| 689 | self._base_warn_line_boards = {} |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 690 | |
| 691 | def PrintFuncSizeDetail(self, fname, old, new): |
| 692 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 |
| 693 | delta, common = [], {} |
| 694 | |
| 695 | for a in old: |
| 696 | if a in new: |
| 697 | common[a] = 1 |
| 698 | |
| 699 | for name in old: |
| 700 | if name not in common: |
| 701 | remove += 1 |
| 702 | down += old[name] |
| 703 | delta.append([-old[name], name]) |
| 704 | |
| 705 | for name in new: |
| 706 | if name not in common: |
| 707 | add += 1 |
| 708 | up += new[name] |
| 709 | delta.append([new[name], name]) |
| 710 | |
| 711 | for name in common: |
| 712 | diff = new.get(name, 0) - old.get(name, 0) |
| 713 | if diff > 0: |
| 714 | grow, up = grow + 1, up + diff |
| 715 | elif diff < 0: |
| 716 | shrink, down = shrink + 1, down - diff |
| 717 | delta.append([diff, name]) |
| 718 | |
| 719 | delta.sort() |
| 720 | delta.reverse() |
| 721 | |
| 722 | args = [add, -remove, grow, -shrink, up, -down, up - down] |
| 723 | if max(args) == 0: |
| 724 | return |
| 725 | args = [self.ColourNum(x) for x in args] |
| 726 | indent = ' ' * 15 |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 727 | Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
| 728 | tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) |
| 729 | Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', |
| 730 | 'delta')) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 731 | for diff, name in delta: |
| 732 | if diff: |
| 733 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 734 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, |
| 735 | old.get(name, '-'), new.get(name,'-'), diff) |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 736 | Print(msg, colour=color) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 737 | |
| 738 | |
| 739 | def PrintSizeDetail(self, target_list, show_bloat): |
| 740 | """Show details size information for each board |
| 741 | |
| 742 | Args: |
| 743 | target_list: List of targets, each a dict containing: |
| 744 | 'target': Target name |
| 745 | 'total_diff': Total difference in bytes across all areas |
| 746 | <part_name>: Difference for that part |
| 747 | show_bloat: Show detail for each function |
| 748 | """ |
| 749 | targets_by_diff = sorted(target_list, reverse=True, |
| 750 | key=lambda x: x['_total_diff']) |
| 751 | for result in targets_by_diff: |
| 752 | printed_target = False |
| 753 | for name in sorted(result): |
| 754 | diff = result[name] |
| 755 | if name.startswith('_'): |
| 756 | continue |
| 757 | if diff != 0: |
| 758 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 759 | msg = ' %s %+d' % (name, diff) |
| 760 | if not printed_target: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 761 | Print('%10s %-15s:' % ('', result['_target']), |
| 762 | newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 763 | printed_target = True |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 764 | Print(msg, colour=color, newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 765 | if printed_target: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 766 | Print() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 767 | if show_bloat: |
| 768 | target = result['_target'] |
| 769 | outcome = result['_outcome'] |
| 770 | base_outcome = self._base_board_dict[target] |
| 771 | for fname in outcome.func_sizes: |
| 772 | self.PrintFuncSizeDetail(fname, |
| 773 | base_outcome.func_sizes[fname], |
| 774 | outcome.func_sizes[fname]) |
| 775 | |
| 776 | |
| 777 | def PrintSizeSummary(self, board_selected, board_dict, show_detail, |
| 778 | show_bloat): |
| 779 | """Print a summary of image sizes broken down by section. |
| 780 | |
| 781 | The summary takes the form of one line per architecture. The |
| 782 | line contains deltas for each of the sections (+ means the section |
| 783 | got bigger, - means smaller). The nunmbers are the average number |
| 784 | of bytes that a board in this section increased by. |
| 785 | |
| 786 | For example: |
| 787 | powerpc: (622 boards) text -0.0 |
| 788 | arm: (285 boards) text -0.0 |
| 789 | nds32: (3 boards) text -8.0 |
| 790 | |
| 791 | Args: |
| 792 | board_selected: Dict containing boards to summarise, keyed by |
| 793 | board.target |
| 794 | board_dict: Dict containing boards for which we built this |
| 795 | commit, keyed by board.target. The value is an Outcome object. |
| 796 | show_detail: Show detail for each board |
| 797 | show_bloat: Show detail for each function |
| 798 | """ |
| 799 | arch_list = {} |
| 800 | arch_count = {} |
| 801 | |
| 802 | # Calculate changes in size for different image parts |
| 803 | # The previous sizes are in Board.sizes, for each board |
| 804 | for target in board_dict: |
| 805 | if target not in board_selected: |
| 806 | continue |
| 807 | base_sizes = self._base_board_dict[target].sizes |
| 808 | outcome = board_dict[target] |
| 809 | sizes = outcome.sizes |
| 810 | |
| 811 | # Loop through the list of images, creating a dict of size |
| 812 | # changes for each image/part. We end up with something like |
| 813 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} |
| 814 | # which means that U-Boot data increased by 5 bytes and SPL |
| 815 | # text decreased by 4. |
| 816 | err = {'_target' : target} |
| 817 | for image in sizes: |
| 818 | if image in base_sizes: |
| 819 | base_image = base_sizes[image] |
| 820 | # Loop through the text, data, bss parts |
| 821 | for part in sorted(sizes[image]): |
| 822 | diff = sizes[image][part] - base_image[part] |
| 823 | col = None |
| 824 | if diff: |
| 825 | if image == 'u-boot': |
| 826 | name = part |
| 827 | else: |
| 828 | name = image + ':' + part |
| 829 | err[name] = diff |
| 830 | arch = board_selected[target].arch |
| 831 | if not arch in arch_count: |
| 832 | arch_count[arch] = 1 |
| 833 | else: |
| 834 | arch_count[arch] += 1 |
| 835 | if not sizes: |
| 836 | pass # Only add to our list when we have some stats |
| 837 | elif not arch in arch_list: |
| 838 | arch_list[arch] = [err] |
| 839 | else: |
| 840 | arch_list[arch].append(err) |
| 841 | |
| 842 | # We now have a list of image size changes sorted by arch |
| 843 | # Print out a summary of these |
| 844 | for arch, target_list in arch_list.iteritems(): |
| 845 | # Get total difference for each type |
| 846 | totals = {} |
| 847 | for result in target_list: |
| 848 | total = 0 |
| 849 | for name, diff in result.iteritems(): |
| 850 | if name.startswith('_'): |
| 851 | continue |
| 852 | total += diff |
| 853 | if name in totals: |
| 854 | totals[name] += diff |
| 855 | else: |
| 856 | totals[name] = diff |
| 857 | result['_total_diff'] = total |
| 858 | result['_outcome'] = board_dict[result['_target']] |
| 859 | |
| 860 | count = len(target_list) |
| 861 | printed_arch = False |
| 862 | for name in sorted(totals): |
| 863 | diff = totals[name] |
| 864 | if diff: |
| 865 | # Display the average difference in this name for this |
| 866 | # architecture |
| 867 | avg_diff = float(diff) / count |
| 868 | color = self.col.RED if avg_diff > 0 else self.col.GREEN |
| 869 | msg = ' %s %+1.1f' % (name, avg_diff) |
| 870 | if not printed_arch: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 871 | Print('%10s: (for %d/%d boards)' % (arch, count, |
| 872 | arch_count[arch]), newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 873 | printed_arch = True |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 874 | Print(msg, colour=color, newline=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 875 | |
| 876 | if printed_arch: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 877 | Print() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 878 | if show_detail: |
| 879 | self.PrintSizeDetail(target_list, show_bloat) |
| 880 | |
| 881 | |
| 882 | def PrintResultSummary(self, board_selected, board_dict, err_lines, |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 883 | err_line_boards, warn_lines, warn_line_boards, |
| 884 | show_sizes, show_detail, show_bloat): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 885 | """Compare results with the base results and display delta. |
| 886 | |
| 887 | Only boards mentioned in board_selected will be considered. This |
| 888 | function is intended to be called repeatedly with the results of |
| 889 | each commit. It therefore shows a 'diff' between what it saw in |
| 890 | the last call and what it sees now. |
| 891 | |
| 892 | Args: |
| 893 | board_selected: Dict containing boards to summarise, keyed by |
| 894 | board.target |
| 895 | board_dict: Dict containing boards for which we built this |
| 896 | commit, keyed by board.target. The value is an Outcome object. |
| 897 | err_lines: A list of errors for this commit, or [] if there is |
| 898 | none, or we don't want to print errors |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 899 | err_line_boards: Dict keyed by error line, containing a list of |
| 900 | the Board objects with that error |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 901 | warn_lines: A list of warnings for this commit, or [] if there is |
| 902 | none, or we don't want to print errors |
| 903 | warn_line_boards: Dict keyed by warning line, containing a list of |
| 904 | the Board objects with that warning |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 905 | show_sizes: Show image size deltas |
| 906 | show_detail: Show detail for each board |
| 907 | show_bloat: Show detail for each function |
| 908 | """ |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 909 | def _BoardList(line, line_boards): |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 910 | """Helper function to get a line of boards containing a line |
| 911 | |
| 912 | Args: |
| 913 | line: Error line to search for |
| 914 | Return: |
| 915 | String containing a list of boards with that error line, or |
| 916 | '' if the user has not requested such a list |
| 917 | """ |
| 918 | if self._list_error_boards: |
| 919 | names = [] |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 920 | for board in line_boards[line]: |
Simon Glass | 85620bb | 2014-10-16 01:05:55 -0600 | [diff] [blame] | 921 | if not board.target in names: |
| 922 | names.append(board.target) |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 923 | names_str = '(%s) ' % ','.join(names) |
| 924 | else: |
| 925 | names_str = '' |
| 926 | return names_str |
| 927 | |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 928 | def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, |
| 929 | char): |
| 930 | better_lines = [] |
| 931 | worse_lines = [] |
| 932 | for line in lines: |
| 933 | if line not in base_lines: |
| 934 | worse_lines.append(char + '+' + |
| 935 | _BoardList(line, line_boards) + line) |
| 936 | for line in base_lines: |
| 937 | if line not in lines: |
| 938 | better_lines.append(char + '-' + |
| 939 | _BoardList(line, base_line_boards) + line) |
| 940 | return better_lines, worse_lines |
| 941 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 942 | better = [] # List of boards fixed since last commit |
| 943 | worse = [] # List of new broken boards since last commit |
| 944 | new = [] # List of boards that didn't exist last time |
| 945 | unknown = [] # List of boards that were not built |
| 946 | |
| 947 | for target in board_dict: |
| 948 | if target not in board_selected: |
| 949 | continue |
| 950 | |
| 951 | # If the board was built last time, add its outcome to a list |
| 952 | if target in self._base_board_dict: |
| 953 | base_outcome = self._base_board_dict[target].rc |
| 954 | outcome = board_dict[target] |
| 955 | if outcome.rc == OUTCOME_UNKNOWN: |
| 956 | unknown.append(target) |
| 957 | elif outcome.rc < base_outcome: |
| 958 | better.append(target) |
| 959 | elif outcome.rc > base_outcome: |
| 960 | worse.append(target) |
| 961 | else: |
| 962 | new.append(target) |
| 963 | |
| 964 | # Get a list of errors that have appeared, and disappeared |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 965 | better_err, worse_err = _CalcErrorDelta(self._base_err_lines, |
| 966 | self._base_err_line_boards, err_lines, err_line_boards, '') |
| 967 | better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, |
| 968 | self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 969 | |
| 970 | # Display results by arch |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 971 | if (better or worse or unknown or new or worse_err or better_err |
| 972 | or worse_warn or better_warn): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 973 | arch_list = {} |
| 974 | self.AddOutcome(board_selected, arch_list, better, '', |
| 975 | self.col.GREEN) |
| 976 | self.AddOutcome(board_selected, arch_list, worse, '+', |
| 977 | self.col.RED) |
| 978 | self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) |
| 979 | if self._show_unknown: |
| 980 | self.AddOutcome(board_selected, arch_list, unknown, '?', |
| 981 | self.col.MAGENTA) |
| 982 | for arch, target_list in arch_list.iteritems(): |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 983 | Print('%10s: %s' % (arch, target_list)) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 984 | self._error_lines += 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 985 | if better_err: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 986 | Print('\n'.join(better_err), colour=self.col.GREEN) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 987 | self._error_lines += 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 988 | if worse_err: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 989 | Print('\n'.join(worse_err), colour=self.col.RED) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 990 | self._error_lines += 1 |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 991 | if better_warn: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 992 | Print('\n'.join(better_warn), colour=self.col.CYAN) |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 993 | self._error_lines += 1 |
| 994 | if worse_warn: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 995 | Print('\n'.join(worse_warn), colour=self.col.MAGENTA) |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 996 | self._error_lines += 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 997 | |
| 998 | if show_sizes: |
| 999 | self.PrintSizeSummary(board_selected, board_dict, show_detail, |
| 1000 | show_bloat) |
| 1001 | |
| 1002 | # Save our updated information for the next call to this function |
| 1003 | self._base_board_dict = board_dict |
| 1004 | self._base_err_lines = err_lines |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1005 | self._base_warn_lines = warn_lines |
| 1006 | self._base_err_line_boards = err_line_boards |
| 1007 | self._base_warn_line_boards = warn_line_boards |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1008 | |
| 1009 | # Get a list of boards that did not get built, if needed |
| 1010 | not_built = [] |
| 1011 | for board in board_selected: |
| 1012 | if not board in board_dict: |
| 1013 | not_built.append(board) |
| 1014 | if not_built: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1015 | Print("Boards not built (%d): %s" % (len(not_built), |
| 1016 | ', '.join(not_built))) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1017 | |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1018 | def ProduceResultSummary(self, commit_upto, commits, board_selected): |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1019 | (board_dict, err_lines, err_line_boards, warn_lines, |
| 1020 | warn_line_boards) = self.GetResultSummary( |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1021 | board_selected, commit_upto, |
| 1022 | read_func_sizes=self._show_bloat) |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1023 | if commits: |
| 1024 | msg = '%02d: %s' % (commit_upto + 1, |
| 1025 | commits[commit_upto].subject) |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1026 | Print(msg, colour=self.col.BLUE) |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1027 | self.PrintResultSummary(board_selected, board_dict, |
Simon Glass | 3394c9f | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1028 | err_lines if self._show_errors else [], err_line_boards, |
Simon Glass | 03749d4 | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1029 | warn_lines if self._show_errors else [], warn_line_boards, |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1030 | self._show_sizes, self._show_detail, self._show_bloat) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1031 | |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1032 | def ShowSummary(self, commits, board_selected): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1033 | """Show a build summary for U-Boot for a given board list. |
| 1034 | |
| 1035 | Reset the result summary, then repeatedly call GetResultSummary on |
| 1036 | each commit's results, then display the differences we see. |
| 1037 | |
| 1038 | Args: |
| 1039 | commit: Commit objects to summarise |
| 1040 | board_selected: Dict containing boards to summarise |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1041 | """ |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1042 | self.commit_count = len(commits) if commits else 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1043 | self.commits = commits |
| 1044 | self.ResetResultSummary(board_selected) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1045 | self._error_lines = 0 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1046 | |
| 1047 | for commit_upto in range(0, self.commit_count, self._step): |
Simon Glass | eb48bbc | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1048 | self.ProduceResultSummary(commit_upto, commits, board_selected) |
Simon Glass | bb4dffb | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1049 | if not self._error_lines: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1050 | Print('(no errors to report)', colour=self.col.GREEN) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1051 | |
| 1052 | |
| 1053 | def SetupBuild(self, board_selected, commits): |
| 1054 | """Set up ready to start a build. |
| 1055 | |
| 1056 | Args: |
| 1057 | board_selected: Selected boards to build |
| 1058 | commits: Selected commits to build |
| 1059 | """ |
| 1060 | # First work out how many commits we will build |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1061 | count = (self.commit_count + self._step - 1) / self._step |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1062 | self.count = len(board_selected) * count |
| 1063 | self.upto = self.warned = self.fail = 0 |
| 1064 | self._timestamps = collections.deque() |
| 1065 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1066 | def GetThreadDir(self, thread_num): |
| 1067 | """Get the directory path to the working dir for a thread. |
| 1068 | |
| 1069 | Args: |
| 1070 | thread_num: Number of thread to check. |
| 1071 | """ |
| 1072 | return os.path.join(self._working_dir, '%02d' % thread_num) |
| 1073 | |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1074 | def _PrepareThread(self, thread_num, setup_git): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1075 | """Prepare the working directory for a thread. |
| 1076 | |
| 1077 | This clones or fetches the repo into the thread's work directory. |
| 1078 | |
| 1079 | Args: |
| 1080 | thread_num: Thread number (0, 1, ...) |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1081 | setup_git: True to set up a git repo clone |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1082 | """ |
| 1083 | thread_dir = self.GetThreadDir(thread_num) |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1084 | builderthread.Mkdir(thread_dir) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1085 | git_dir = os.path.join(thread_dir, '.git') |
| 1086 | |
| 1087 | # Clone the repo if it doesn't already exist |
| 1088 | # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so |
| 1089 | # we have a private index but uses the origin repo's contents? |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1090 | if setup_git and self.git_dir: |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1091 | src_dir = os.path.abspath(self.git_dir) |
| 1092 | if os.path.exists(git_dir): |
| 1093 | gitutil.Fetch(git_dir, thread_dir) |
| 1094 | else: |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1095 | Print('Cloning repo for thread %d' % thread_num) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1096 | gitutil.Clone(src_dir, thread_dir) |
| 1097 | |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1098 | def _PrepareWorkingSpace(self, max_threads, setup_git): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1099 | """Prepare the working directory for use. |
| 1100 | |
| 1101 | Set up the git repo for each thread. |
| 1102 | |
| 1103 | Args: |
| 1104 | max_threads: Maximum number of threads we expect to need. |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1105 | setup_git: True to set up a git repo clone |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1106 | """ |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1107 | builderthread.Mkdir(self._working_dir) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1108 | for thread in range(max_threads): |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1109 | self._PrepareThread(thread, setup_git) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1110 | |
| 1111 | def _PrepareOutputSpace(self): |
| 1112 | """Get the output directories ready to receive files. |
| 1113 | |
| 1114 | We delete any output directories which look like ones we need to |
| 1115 | create. Having left over directories is confusing when the user wants |
| 1116 | to check the output manually. |
| 1117 | """ |
| 1118 | dir_list = [] |
| 1119 | for commit_upto in range(self.commit_count): |
| 1120 | dir_list.append(self._GetOutputDir(commit_upto)) |
| 1121 | |
| 1122 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): |
| 1123 | if dirname not in dir_list: |
| 1124 | shutil.rmtree(dirname) |
| 1125 | |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1126 | def BuildBoards(self, commits, board_selected, keep_outputs, verbose): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1127 | """Build all commits for a list of boards |
| 1128 | |
| 1129 | Args: |
| 1130 | commits: List of commits to be build, each a Commit object |
| 1131 | boards_selected: Dict of selected boards, key is target name, |
| 1132 | value is Board object |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1133 | keep_outputs: True to save build output files |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1134 | verbose: Display build results as they are completed |
Simon Glass | c2f9107 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 1135 | Returns: |
| 1136 | Tuple containing: |
| 1137 | - number of boards that failed to build |
| 1138 | - number of boards that issued warnings |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1139 | """ |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1140 | self.commit_count = len(commits) if commits else 1 |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1141 | self.commits = commits |
Simon Glass | 78e418e | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1142 | self._verbose = verbose |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1143 | |
| 1144 | self.ResetResultSummary(board_selected) |
Thierry Reding | 336d5bd | 2014-08-19 10:22:39 +0200 | [diff] [blame] | 1145 | builderthread.Mkdir(self.base_dir, parents = True) |
Simon Glass | d326ad7 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1146 | self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), |
| 1147 | commits is not None) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1148 | self._PrepareOutputSpace() |
| 1149 | self.SetupBuild(board_selected, commits) |
| 1150 | self.ProcessResult(None) |
| 1151 | |
| 1152 | # Create jobs to build all commits for each board |
| 1153 | for brd in board_selected.itervalues(): |
Simon Glass | 4a1e88b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1154 | job = builderthread.BuilderJob() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1155 | job.board = brd |
| 1156 | job.commits = commits |
| 1157 | job.keep_outputs = keep_outputs |
| 1158 | job.step = self._step |
| 1159 | self.queue.put(job) |
| 1160 | |
| 1161 | # Wait until all jobs are started |
| 1162 | self.queue.join() |
| 1163 | |
| 1164 | # Wait until we have processed all output |
| 1165 | self.out_queue.join() |
Simon Glass | 4433aa9 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1166 | Print() |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1167 | self.ClearLine(0) |
Simon Glass | c2f9107 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 1168 | return (self.fail, self.warned) |