blob: 216012d00168f25b4ce6e41c40212f2b90535f73 [file] [log] [blame]
Tom Rini10e47792018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassc05694f2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
Simon Glassc05694f2013-04-03 11:07:16 +00004
5import multiprocessing
6import os
Simon Glassa10ebe12014-09-05 19:00:18 -06007import shutil
Simon Glassc05694f2013-04-03 11:07:16 +00008import sys
9
10import board
11import bsettings
12from builder import Builder
13import gitutil
14import patchstream
15import terminal
Simon Glassed098bb2014-09-05 19:00:13 -060016from terminal import Print
Simon Glassc05694f2013-04-03 11:07:16 +000017import toolchain
Masahiro Yamada1fe610d2014-07-22 11:19:09 +090018import command
Masahiro Yamadae9bc8d22014-07-30 14:08:22 +090019import subprocess
Simon Glassc05694f2013-04-03 11:07:16 +000020
21def GetPlural(count):
22 """Returns a plural 's' if count is not 1"""
23 return 's' if count != 1 else ''
24
Simon Glassd326ad72014-08-09 15:32:59 -060025def GetActionSummary(is_summary, commits, selected, options):
Simon Glassc05694f2013-04-03 11:07:16 +000026 """Return a string summarising the intended action.
27
28 Returns:
29 Summary string.
30 """
Simon Glassd326ad72014-08-09 15:32:59 -060031 if commits:
32 count = len(commits)
Simon Glassc78ed662019-10-31 07:42:53 -060033 count = (count + options.step - 1) // options.step
Simon Glassd326ad72014-08-09 15:32:59 -060034 commit_str = '%d commit%s' % (count, GetPlural(count))
35 else:
36 commit_str = 'current source'
37 str = '%s %s for %d boards' % (
38 'Summary of' if is_summary else 'Building', commit_str,
Simon Glassc05694f2013-04-03 11:07:16 +000039 len(selected))
40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
42 return str
43
Simon Glassd9eb9f02018-06-11 23:26:46 -060044def ShowActions(series, why_selected, boards_selected, builder, options,
45 board_warnings):
Simon Glassc05694f2013-04-03 11:07:16 +000046 """Display a list of actions that we would take, if not a dry run.
47
48 Args:
49 series: Series object
50 why_selected: Dictionary where each key is a buildman argument
Simon Glass6af145f2017-01-23 05:38:56 -070051 provided by the user, and the value is the list of boards
52 brought in by that argument. For example, 'arm' might bring
53 in 400 boards, so in this case the key would be 'arm' and
Simon Glassc05694f2013-04-03 11:07:16 +000054 the value would be a list of board names.
55 boards_selected: Dict of selected boards, key is target name,
56 value is Board object
57 builder: The builder that will be used to build the commits
58 options: Command line options object
Simon Glassd9eb9f02018-06-11 23:26:46 -060059 board_warnings: List of warnings obtained from board selected
Simon Glassc05694f2013-04-03 11:07:16 +000060 """
61 col = terminal.Color()
Simon Glassc78ed662019-10-31 07:42:53 -060062 print('Dry run, so not doing much. But I would do this:')
63 print()
Simon Glassd326ad72014-08-09 15:32:59 -060064 if series:
65 commits = series.commits
66 else:
67 commits = None
Simon Glassc78ed662019-10-31 07:42:53 -060068 print(GetActionSummary(False, commits, boards_selected,
69 options))
70 print('Build directory: %s' % builder.base_dir)
Simon Glassd326ad72014-08-09 15:32:59 -060071 if commits:
72 for upto in range(0, len(series.commits), options.step):
73 commit = series.commits[upto]
Simon Glassc78ed662019-10-31 07:42:53 -060074 print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
75 print(commit.subject)
76 print()
Simon Glassc05694f2013-04-03 11:07:16 +000077 for arg in why_selected:
78 if arg != 'all':
Simon Glassc78ed662019-10-31 07:42:53 -060079 print(arg, ': %d boards' % len(why_selected[arg]))
Simon Glass6af145f2017-01-23 05:38:56 -070080 if options.verbose:
Simon Glassc78ed662019-10-31 07:42:53 -060081 print(' %s' % ' '.join(why_selected[arg]))
82 print(('Total boards to build for each commit: %d\n' %
83 len(why_selected['all'])))
Simon Glassd9eb9f02018-06-11 23:26:46 -060084 if board_warnings:
85 for warning in board_warnings:
Simon Glassc78ed662019-10-31 07:42:53 -060086 print(col.Color(col.YELLOW, warning))
Simon Glassc05694f2013-04-03 11:07:16 +000087
Lothar Waßmannce6df922018-04-08 05:14:11 -060088def CheckOutputDir(output_dir):
89 """Make sure that the output directory is not within the current directory
90
91 If we try to use an output directory which is within the current directory
92 (which is assumed to hold the U-Boot source) we may end up deleting the
93 U-Boot source code. Detect this and print an error in this case.
94
95 Args:
96 output_dir: Output directory path to check
97 """
98 path = os.path.realpath(output_dir)
99 cwd_path = os.path.realpath('.')
100 while True:
101 if os.path.realpath(path) == cwd_path:
Chris Packhamdf4e8052019-01-18 20:40:29 +1300102 Print("Cannot use output directory '%s' since it is within the current directory '%s'" %
Lothar Waßmannce6df922018-04-08 05:14:11 -0600103 (path, cwd_path))
104 sys.exit(1)
105 parent = os.path.dirname(path)
106 if parent == path:
107 break
108 path = parent
109
Simon Glassa10ebe12014-09-05 19:00:18 -0600110def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
111 clean_dir=False):
Simon Glassc05694f2013-04-03 11:07:16 +0000112 """The main control code for buildman
113
114 Args:
115 options: Command line options object
116 args: Command line arguments (list of strings)
Simon Glassed098bb2014-09-05 19:00:13 -0600117 toolchains: Toolchains to use - this should be a Toolchains()
118 object. If None, then it will be created and scanned
119 make_func: Make function to use for the builder. This is called
120 to execute 'make'. If this is None, the normal function
121 will be used, which calls the 'make' tool with suitable
122 arguments. This setting is useful for tests.
Simon Glasscbd36582014-09-05 19:00:16 -0600123 board: Boards() object to use, containing a list of available
124 boards. If this is None it will be created and scanned.
Simon Glassc05694f2013-04-03 11:07:16 +0000125 """
Simon Glassa10ebe12014-09-05 19:00:18 -0600126 global builder
127
Simon Glassca9b06e2014-09-05 19:00:11 -0600128 if options.full_help:
129 pager = os.getenv('PAGER')
130 if not pager:
131 pager = 'more'
Simon Glassc29101f2016-03-06 19:45:34 -0700132 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
133 'README')
Simon Glassca9b06e2014-09-05 19:00:11 -0600134 command.Run(pager, fname)
135 return 0
136
Simon Glassc05694f2013-04-03 11:07:16 +0000137 gitutil.Setup()
Simon Glass9f1ba0f2016-07-27 20:33:02 -0600138 col = terminal.Color()
Simon Glassc05694f2013-04-03 11:07:16 +0000139
Simon Glassc05694f2013-04-03 11:07:16 +0000140 options.git_dir = os.path.join(options.git, '.git')
141
Simon Glassa3d9b4f2016-07-27 20:33:04 -0600142 no_toolchains = toolchains is None
143 if no_toolchains:
Simon Glassf77ca5b2019-01-07 16:44:20 -0700144 toolchains = toolchain.Toolchains(options.override_toolchain)
Simon Glassc05694f2013-04-03 11:07:16 +0000145
Simon Glass7e803e12014-12-01 17:34:06 -0700146 if options.fetch_arch:
147 if options.fetch_arch == 'list':
148 sorted_list = toolchains.ListArchs()
Simon Glassc78ed662019-10-31 07:42:53 -0600149 print(col.Color(col.BLUE, 'Available architectures: %s\n' %
150 ' '.join(sorted_list)))
Simon Glass7e803e12014-12-01 17:34:06 -0700151 return 0
152 else:
153 fetch_arch = options.fetch_arch
154 if fetch_arch == 'all':
155 fetch_arch = ','.join(toolchains.ListArchs())
Simon Glassc78ed662019-10-31 07:42:53 -0600156 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
157 fetch_arch))
Simon Glass7e803e12014-12-01 17:34:06 -0700158 for arch in fetch_arch.split(','):
Simon Glassc78ed662019-10-31 07:42:53 -0600159 print()
Simon Glass7e803e12014-12-01 17:34:06 -0700160 ret = toolchains.FetchAndInstall(arch)
161 if ret:
162 return ret
163 return 0
164
Simon Glassa3d9b4f2016-07-27 20:33:04 -0600165 if no_toolchains:
166 toolchains.GetSettings()
Simon Glass74579fc2018-11-06 16:02:10 -0700167 toolchains.Scan(options.list_tool_chains and options.verbose)
Simon Glassa3d9b4f2016-07-27 20:33:04 -0600168 if options.list_tool_chains:
169 toolchains.List()
Simon Glassc78ed662019-10-31 07:42:53 -0600170 print()
Simon Glassa3d9b4f2016-07-27 20:33:04 -0600171 return 0
172
Simon Glassc05694f2013-04-03 11:07:16 +0000173 # Work out how many commits to build. We want to build everything on the
174 # branch. We also build the upstream commit as a control so we can see
175 # problems introduced by the first commit on the branch.
Simon Glassc05694f2013-04-03 11:07:16 +0000176 count = options.count
Simon Glass5eeef462014-12-01 17:33:57 -0700177 has_range = options.branch and '..' in options.branch
Simon Glassc05694f2013-04-03 11:07:16 +0000178 if count == -1:
179 if not options.branch:
Simon Glassd326ad72014-08-09 15:32:59 -0600180 count = 1
181 else:
Simon Glass5eeef462014-12-01 17:33:57 -0700182 if has_range:
183 count, msg = gitutil.CountCommitsInRange(options.git_dir,
184 options.branch)
185 else:
186 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
187 options.branch)
Simon Glassd326ad72014-08-09 15:32:59 -0600188 if count is None:
Simon Glassf204ab12014-12-01 17:33:54 -0700189 sys.exit(col.Color(col.RED, msg))
Simon Glass5eeef462014-12-01 17:33:57 -0700190 elif count == 0:
191 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
192 options.branch))
Simon Glassf204ab12014-12-01 17:33:54 -0700193 if msg:
Simon Glassc78ed662019-10-31 07:42:53 -0600194 print(col.Color(col.YELLOW, msg))
Simon Glassd326ad72014-08-09 15:32:59 -0600195 count += 1 # Build upstream commit also
Simon Glassc05694f2013-04-03 11:07:16 +0000196
197 if not count:
198 str = ("No commits found to process in branch '%s': "
199 "set branch's upstream or use -c flag" % options.branch)
Masahiro Yamada880828d2014-08-16 00:59:26 +0900200 sys.exit(col.Color(col.RED, str))
Simon Glassc05694f2013-04-03 11:07:16 +0000201
202 # Work out what subset of the boards we are building
Simon Glasscbd36582014-09-05 19:00:16 -0600203 if not boards:
Bin Meng0733e202019-10-28 07:24:59 -0700204 board_file = os.path.join(options.output_dir, 'boards.cfg')
205 genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
206 status = subprocess.call([genboardscfg, '-o', board_file])
Simon Glasscbd36582014-09-05 19:00:16 -0600207 if status != 0:
Bin Meng0733e202019-10-28 07:24:59 -0700208 sys.exit("Failed to generate boards.cfg")
Masahiro Yamadae9bc8d22014-07-30 14:08:22 +0900209
Simon Glasscbd36582014-09-05 19:00:16 -0600210 boards = board.Boards()
Bin Meng0733e202019-10-28 07:24:59 -0700211 boards.ReadBoards(board_file)
Simon Glass924c73a2014-08-28 09:43:41 -0600212
213 exclude = []
214 if options.exclude:
215 for arg in options.exclude:
216 exclude += arg.split(',')
217
Simon Glassd9eb9f02018-06-11 23:26:46 -0600218
219 if options.boards:
220 requested_boards = []
221 for b in options.boards:
222 requested_boards += b.split(',')
223 else:
224 requested_boards = None
225 why_selected, board_warnings = boards.SelectBoards(args, exclude,
226 requested_boards)
Simon Glassc05694f2013-04-03 11:07:16 +0000227 selected = boards.GetSelected()
228 if not len(selected):
Masahiro Yamada880828d2014-08-16 00:59:26 +0900229 sys.exit(col.Color(col.RED, 'No matching boards found'))
Simon Glassc05694f2013-04-03 11:07:16 +0000230
231 # Read the metadata from the commits. First look at the upstream commit,
232 # then the ones in the branch. We would like to do something like
233 # upstream/master~..branch but that isn't possible if upstream/master is
234 # a merge commit (it will list all the commits that form part of the
235 # merge)
Simon Glass359b55a62014-09-05 19:00:23 -0600236 # Conflicting tags are not a problem for buildman, since it does not use
237 # them. For example, Series-version is not useful for buildman. On the
238 # other hand conflicting tags will cause an error. So allow later tags
239 # to overwrite earlier ones by setting allow_overwrite=True
Simon Glassd326ad72014-08-09 15:32:59 -0600240 if options.branch:
Simon Glass16a52882014-08-09 15:33:09 -0600241 if count == -1:
Simon Glass5eeef462014-12-01 17:33:57 -0700242 if has_range:
243 range_expr = options.branch
244 else:
245 range_expr = gitutil.GetRangeInBranch(options.git_dir,
246 options.branch)
Simon Glass16a52882014-08-09 15:33:09 -0600247 upstream_commit = gitutil.GetUpstream(options.git_dir,
248 options.branch)
249 series = patchstream.GetMetaDataForList(upstream_commit,
Simon Glass359b55a62014-09-05 19:00:23 -0600250 options.git_dir, 1, series=None, allow_overwrite=True)
Simon Glassd326ad72014-08-09 15:32:59 -0600251
Simon Glass16a52882014-08-09 15:33:09 -0600252 series = patchstream.GetMetaDataForList(range_expr,
Simon Glass359b55a62014-09-05 19:00:23 -0600253 options.git_dir, None, series, allow_overwrite=True)
Simon Glass16a52882014-08-09 15:33:09 -0600254 else:
255 # Honour the count
256 series = patchstream.GetMetaDataForList(options.branch,
Simon Glass359b55a62014-09-05 19:00:23 -0600257 options.git_dir, count, series=None, allow_overwrite=True)
Simon Glassd326ad72014-08-09 15:32:59 -0600258 else:
259 series = None
Simon Glass6af145f2017-01-23 05:38:56 -0700260 if not options.dry_run:
261 options.verbose = True
262 if not options.summary:
263 options.show_errors = True
Simon Glassc05694f2013-04-03 11:07:16 +0000264
265 # By default we have one thread per CPU. But if there are not enough jobs
266 # we can have fewer threads and use a high '-j' value for make.
267 if not options.threads:
268 options.threads = min(multiprocessing.cpu_count(), len(selected))
269 if not options.jobs:
270 options.jobs = max(1, (multiprocessing.cpu_count() +
Simon Glassc78ed662019-10-31 07:42:53 -0600271 len(selected) - 1) // len(selected))
Simon Glassc05694f2013-04-03 11:07:16 +0000272
273 if not options.step:
274 options.step = len(series.commits) - 1
275
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900276 gnu_make = command.Output(os.path.join(options.git,
Simon Glassc55e0562016-07-25 18:59:00 -0600277 'scripts/show-gnu-make'), raise_on_error=False).rstrip()
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900278 if not gnu_make:
Masahiro Yamada880828d2014-08-16 00:59:26 +0900279 sys.exit('GNU Make not found')
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900280
Simon Glassdbc01c72014-12-01 17:33:52 -0700281 # Create a new builder with the selected options.
282 output_dir = options.output_dir
Simon Glassd326ad72014-08-09 15:32:59 -0600283 if options.branch:
Simon Glass4aeceb92014-09-05 19:00:22 -0600284 dirname = options.branch.replace('/', '_')
Simon Glasse87bde12014-12-01 17:33:55 -0700285 # As a special case allow the board directory to be placed in the
286 # output directory itself rather than any subdirectory.
287 if not options.no_subdirs:
288 output_dir = os.path.join(options.output_dir, dirname)
Lothar Waßmannce6df922018-04-08 05:14:11 -0600289 if clean_dir and os.path.exists(output_dir):
290 shutil.rmtree(output_dir)
291 CheckOutputDir(output_dir)
Simon Glassc05694f2013-04-03 11:07:16 +0000292 builder = Builder(toolchains, output_dir, options.git_dir,
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900293 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
Simon Glasse87bde12014-12-01 17:33:55 -0700294 show_unknown=options.show_unknown, step=options.step,
Simon Glass655b6102014-12-01 17:34:07 -0700295 no_subdirs=options.no_subdirs, full_path=options.full_path,
Stephen Warren97c96902016-04-11 10:48:44 -0600296 verbose_build=options.verbose_build,
297 incremental=options.incremental,
Simon Glass739e8512016-11-13 14:25:51 -0700298 per_board_out_dir=options.per_board_out_dir,
Simon Glasscde5c302016-11-13 14:25:53 -0700299 config_only=options.config_only,
Daniel Schwierzeck20e2ea92018-01-26 16:31:05 +0100300 squash_config_y=not options.preserve_config_y,
301 warnings_as_errors=options.warnings_as_errors)
Simon Glassc05694f2013-04-03 11:07:16 +0000302 builder.force_config_on_failure = not options.quick
Simon Glassed098bb2014-09-05 19:00:13 -0600303 if make_func:
304 builder.do_make = make_func
Simon Glassc05694f2013-04-03 11:07:16 +0000305
306 # For a dry run, just show our actions as a sanity check
307 if options.dry_run:
Simon Glassd9eb9f02018-06-11 23:26:46 -0600308 ShowActions(series, why_selected, selected, builder, options,
309 board_warnings)
Simon Glassc05694f2013-04-03 11:07:16 +0000310 else:
311 builder.force_build = options.force_build
Simon Glass7041c392014-07-13 12:22:31 -0600312 builder.force_build_failures = options.force_build_failures
Simon Glassf3018b7a2014-07-14 17:51:02 -0600313 builder.force_reconfig = options.force_reconfig
Simon Glass38df2e22014-07-14 17:51:03 -0600314 builder.in_tree = options.in_tree
Simon Glassc05694f2013-04-03 11:07:16 +0000315
316 # Work out which boards to build
317 board_selected = boards.GetSelectedDict()
318
Simon Glassd326ad72014-08-09 15:32:59 -0600319 if series:
320 commits = series.commits
Simon Glassa10ebe12014-09-05 19:00:18 -0600321 # Number the commits for test purposes
322 for commit in range(len(commits)):
323 commits[commit].sequence = commit
Simon Glassd326ad72014-08-09 15:32:59 -0600324 else:
325 commits = None
326
Simon Glassed098bb2014-09-05 19:00:13 -0600327 Print(GetActionSummary(options.summary, commits, board_selected,
328 options))
Simon Glassc05694f2013-04-03 11:07:16 +0000329
Simon Glass232d8502014-09-14 20:23:16 -0600330 # We can't show function sizes without board details at present
331 if options.show_bloat:
332 options.show_detail = True
Simon Glasseb48bbc2014-08-09 15:33:02 -0600333 builder.SetDisplayOptions(options.show_errors, options.show_sizes,
Simon Glass3394c9f2014-08-28 09:43:43 -0600334 options.show_detail, options.show_bloat,
Simon Glassdb17fb82015-02-05 22:06:15 -0700335 options.list_error_boards,
Alex Kiernan4059e302018-05-31 04:48:34 +0000336 options.show_config,
337 options.show_environment)
Simon Glassc05694f2013-04-03 11:07:16 +0000338 if options.summary:
Simon Glasseb48bbc2014-08-09 15:33:02 -0600339 builder.ShowSummary(commits, board_selected)
Simon Glassc05694f2013-04-03 11:07:16 +0000340 else:
Simon Glassc2f91072014-08-28 09:43:39 -0600341 fail, warned = builder.BuildBoards(commits, board_selected,
Simon Glass78e418e2014-08-09 15:33:03 -0600342 options.keep_outputs, options.verbose)
Simon Glassc2f91072014-08-28 09:43:39 -0600343 if fail:
344 return 128
345 elif warned:
346 return 129
347 return 0