blob: b86d7b3c1f901f2598778e0b94e4543d863fe758 [file] [log] [blame]
Simon Glassc05694f2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
Wolfgang Denkd79de1d2013-07-08 09:37:19 +02003# SPDX-License-Identifier: GPL-2.0+
Simon Glassc05694f2013-04-03 11:07:16 +00004#
5
6import multiprocessing
7import os
Simon Glassa10ebe12014-09-05 19:00:18 -06008import shutil
Simon Glassc05694f2013-04-03 11:07:16 +00009import sys
10
11import board
12import bsettings
13from builder import Builder
14import gitutil
15import patchstream
16import terminal
Simon Glassed098bb2014-09-05 19:00:13 -060017from terminal import Print
Simon Glassc05694f2013-04-03 11:07:16 +000018import toolchain
Masahiro Yamada1fe610d2014-07-22 11:19:09 +090019import command
Masahiro Yamadae9bc8d22014-07-30 14:08:22 +090020import subprocess
Simon Glassc05694f2013-04-03 11:07:16 +000021
22def GetPlural(count):
23 """Returns a plural 's' if count is not 1"""
24 return 's' if count != 1 else ''
25
Simon Glassd326ad72014-08-09 15:32:59 -060026def GetActionSummary(is_summary, commits, selected, options):
Simon Glassc05694f2013-04-03 11:07:16 +000027 """Return a string summarising the intended action.
28
29 Returns:
30 Summary string.
31 """
Simon Glassd326ad72014-08-09 15:32:59 -060032 if commits:
33 count = len(commits)
34 count = (count + options.step - 1) / options.step
35 commit_str = '%d commit%s' % (count, GetPlural(count))
36 else:
37 commit_str = 'current source'
38 str = '%s %s for %d boards' % (
39 'Summary of' if is_summary else 'Building', commit_str,
Simon Glassc05694f2013-04-03 11:07:16 +000040 len(selected))
41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
43 return str
44
45def ShowActions(series, why_selected, boards_selected, builder, options):
46 """Display a list of actions that we would take, if not a dry run.
47
48 Args:
49 series: Series object
50 why_selected: Dictionary where each key is a buildman argument
51 provided by the user, and the value is the boards brought
52 in by that argument. For example, 'arm' might bring in
53 400 boards, so in this case the key would be 'arm' and
54 the value would be a list of board names.
55 boards_selected: Dict of selected boards, key is target name,
56 value is Board object
57 builder: The builder that will be used to build the commits
58 options: Command line options object
59 """
60 col = terminal.Color()
61 print 'Dry run, so not doing much. But I would do this:'
62 print
Simon Glassd326ad72014-08-09 15:32:59 -060063 if series:
64 commits = series.commits
65 else:
66 commits = None
67 print GetActionSummary(False, commits, boards_selected,
Simon Glassc05694f2013-04-03 11:07:16 +000068 options)
69 print 'Build directory: %s' % builder.base_dir
Simon Glassd326ad72014-08-09 15:32:59 -060070 if commits:
71 for upto in range(0, len(series.commits), options.step):
72 commit = series.commits[upto]
Simon Glasse2fb4192014-10-15 02:27:00 -060073 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
Simon Glassd326ad72014-08-09 15:32:59 -060074 print commit.subject
Simon Glassc05694f2013-04-03 11:07:16 +000075 print
76 for arg in why_selected:
77 if arg != 'all':
78 print arg, ': %d boards' % why_selected[arg]
79 print ('Total boards to build for each commit: %d\n' %
80 why_selected['all'])
81
Simon Glassa10ebe12014-09-05 19:00:18 -060082def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
83 clean_dir=False):
Simon Glassc05694f2013-04-03 11:07:16 +000084 """The main control code for buildman
85
86 Args:
87 options: Command line options object
88 args: Command line arguments (list of strings)
Simon Glassed098bb2014-09-05 19:00:13 -060089 toolchains: Toolchains to use - this should be a Toolchains()
90 object. If None, then it will be created and scanned
91 make_func: Make function to use for the builder. This is called
92 to execute 'make'. If this is None, the normal function
93 will be used, which calls the 'make' tool with suitable
94 arguments. This setting is useful for tests.
Simon Glasscbd36582014-09-05 19:00:16 -060095 board: Boards() object to use, containing a list of available
96 boards. If this is None it will be created and scanned.
Simon Glassc05694f2013-04-03 11:07:16 +000097 """
Simon Glassa10ebe12014-09-05 19:00:18 -060098 global builder
99
Simon Glassca9b06e2014-09-05 19:00:11 -0600100 if options.full_help:
101 pager = os.getenv('PAGER')
102 if not pager:
103 pager = 'more'
Simon Glassc29101f2016-03-06 19:45:34 -0700104 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
105 'README')
Simon Glassca9b06e2014-09-05 19:00:11 -0600106 command.Run(pager, fname)
107 return 0
108
Simon Glassc05694f2013-04-03 11:07:16 +0000109 gitutil.Setup()
Simon Glass9f1ba0f2016-07-27 20:33:02 -0600110 col = terminal.Color()
Simon Glassc05694f2013-04-03 11:07:16 +0000111
Simon Glassc05694f2013-04-03 11:07:16 +0000112 options.git_dir = os.path.join(options.git, '.git')
113
Simon Glassa3d9b4f2016-07-27 20:33:04 -0600114 no_toolchains = toolchains is None
115 if no_toolchains:
Simon Glassed098bb2014-09-05 19:00:13 -0600116 toolchains = toolchain.Toolchains()
Simon Glassc05694f2013-04-03 11:07:16 +0000117
Simon Glass7e803e12014-12-01 17:34:06 -0700118 if options.fetch_arch:
119 if options.fetch_arch == 'list':
120 sorted_list = toolchains.ListArchs()
Simon Glass9f1ba0f2016-07-27 20:33:02 -0600121 print col.Color(col.BLUE, 'Available architectures: %s\n' %
122 ' '.join(sorted_list))
Simon Glass7e803e12014-12-01 17:34:06 -0700123 return 0
124 else:
125 fetch_arch = options.fetch_arch
126 if fetch_arch == 'all':
127 fetch_arch = ','.join(toolchains.ListArchs())
Simon Glass9f1ba0f2016-07-27 20:33:02 -0600128 print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
129 fetch_arch)
Simon Glass7e803e12014-12-01 17:34:06 -0700130 for arch in fetch_arch.split(','):
Simon Glass9f1ba0f2016-07-27 20:33:02 -0600131 print
Simon Glass7e803e12014-12-01 17:34:06 -0700132 ret = toolchains.FetchAndInstall(arch)
133 if ret:
134 return ret
135 return 0
136
Simon Glassa3d9b4f2016-07-27 20:33:04 -0600137 if no_toolchains:
138 toolchains.GetSettings()
139 toolchains.Scan(options.list_tool_chains)
140 if options.list_tool_chains:
141 toolchains.List()
142 print
143 return 0
144
Simon Glassc05694f2013-04-03 11:07:16 +0000145 # Work out how many commits to build. We want to build everything on the
146 # branch. We also build the upstream commit as a control so we can see
147 # problems introduced by the first commit on the branch.
Simon Glassc05694f2013-04-03 11:07:16 +0000148 count = options.count
Simon Glass5eeef462014-12-01 17:33:57 -0700149 has_range = options.branch and '..' in options.branch
Simon Glassc05694f2013-04-03 11:07:16 +0000150 if count == -1:
151 if not options.branch:
Simon Glassd326ad72014-08-09 15:32:59 -0600152 count = 1
153 else:
Simon Glass5eeef462014-12-01 17:33:57 -0700154 if has_range:
155 count, msg = gitutil.CountCommitsInRange(options.git_dir,
156 options.branch)
157 else:
158 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
159 options.branch)
Simon Glassd326ad72014-08-09 15:32:59 -0600160 if count is None:
Simon Glassf204ab12014-12-01 17:33:54 -0700161 sys.exit(col.Color(col.RED, msg))
Simon Glass5eeef462014-12-01 17:33:57 -0700162 elif count == 0:
163 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
164 options.branch))
Simon Glassf204ab12014-12-01 17:33:54 -0700165 if msg:
166 print col.Color(col.YELLOW, msg)
Simon Glassd326ad72014-08-09 15:32:59 -0600167 count += 1 # Build upstream commit also
Simon Glassc05694f2013-04-03 11:07:16 +0000168
169 if not count:
170 str = ("No commits found to process in branch '%s': "
171 "set branch's upstream or use -c flag" % options.branch)
Masahiro Yamada880828d2014-08-16 00:59:26 +0900172 sys.exit(col.Color(col.RED, str))
Simon Glassc05694f2013-04-03 11:07:16 +0000173
174 # Work out what subset of the boards we are building
Simon Glasscbd36582014-09-05 19:00:16 -0600175 if not boards:
176 board_file = os.path.join(options.git, 'boards.cfg')
177 status = subprocess.call([os.path.join(options.git,
178 'tools/genboardscfg.py')])
179 if status != 0:
180 sys.exit("Failed to generate boards.cfg")
Masahiro Yamadae9bc8d22014-07-30 14:08:22 +0900181
Simon Glasscbd36582014-09-05 19:00:16 -0600182 boards = board.Boards()
183 boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
Simon Glass924c73a2014-08-28 09:43:41 -0600184
185 exclude = []
186 if options.exclude:
187 for arg in options.exclude:
188 exclude += arg.split(',')
189
190 why_selected = boards.SelectBoards(args, exclude)
Simon Glassc05694f2013-04-03 11:07:16 +0000191 selected = boards.GetSelected()
192 if not len(selected):
Masahiro Yamada880828d2014-08-16 00:59:26 +0900193 sys.exit(col.Color(col.RED, 'No matching boards found'))
Simon Glassc05694f2013-04-03 11:07:16 +0000194
195 # Read the metadata from the commits. First look at the upstream commit,
196 # then the ones in the branch. We would like to do something like
197 # upstream/master~..branch but that isn't possible if upstream/master is
198 # a merge commit (it will list all the commits that form part of the
199 # merge)
Simon Glass359b55a62014-09-05 19:00:23 -0600200 # Conflicting tags are not a problem for buildman, since it does not use
201 # them. For example, Series-version is not useful for buildman. On the
202 # other hand conflicting tags will cause an error. So allow later tags
203 # to overwrite earlier ones by setting allow_overwrite=True
Simon Glassd326ad72014-08-09 15:32:59 -0600204 if options.branch:
Simon Glass16a52882014-08-09 15:33:09 -0600205 if count == -1:
Simon Glass5eeef462014-12-01 17:33:57 -0700206 if has_range:
207 range_expr = options.branch
208 else:
209 range_expr = gitutil.GetRangeInBranch(options.git_dir,
210 options.branch)
Simon Glass16a52882014-08-09 15:33:09 -0600211 upstream_commit = gitutil.GetUpstream(options.git_dir,
212 options.branch)
213 series = patchstream.GetMetaDataForList(upstream_commit,
Simon Glass359b55a62014-09-05 19:00:23 -0600214 options.git_dir, 1, series=None, allow_overwrite=True)
Simon Glassd326ad72014-08-09 15:32:59 -0600215
Simon Glass16a52882014-08-09 15:33:09 -0600216 series = patchstream.GetMetaDataForList(range_expr,
Simon Glass359b55a62014-09-05 19:00:23 -0600217 options.git_dir, None, series, allow_overwrite=True)
Simon Glass16a52882014-08-09 15:33:09 -0600218 else:
219 # Honour the count
220 series = patchstream.GetMetaDataForList(options.branch,
Simon Glass359b55a62014-09-05 19:00:23 -0600221 options.git_dir, count, series=None, allow_overwrite=True)
Simon Glassd326ad72014-08-09 15:32:59 -0600222 else:
223 series = None
Simon Glass78e418e2014-08-09 15:33:03 -0600224 options.verbose = True
Simon Glassd1f00562014-10-15 14:37:25 +0200225 if not options.summary:
226 options.show_errors = True
Simon Glassc05694f2013-04-03 11:07:16 +0000227
228 # By default we have one thread per CPU. But if there are not enough jobs
229 # we can have fewer threads and use a high '-j' value for make.
230 if not options.threads:
231 options.threads = min(multiprocessing.cpu_count(), len(selected))
232 if not options.jobs:
233 options.jobs = max(1, (multiprocessing.cpu_count() +
234 len(selected) - 1) / len(selected))
235
236 if not options.step:
237 options.step = len(series.commits) - 1
238
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900239 gnu_make = command.Output(os.path.join(options.git,
240 'scripts/show-gnu-make')).rstrip()
241 if not gnu_make:
Masahiro Yamada880828d2014-08-16 00:59:26 +0900242 sys.exit('GNU Make not found')
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900243
Simon Glassdbc01c72014-12-01 17:33:52 -0700244 # Create a new builder with the selected options.
245 output_dir = options.output_dir
Simon Glassd326ad72014-08-09 15:32:59 -0600246 if options.branch:
Simon Glass4aeceb92014-09-05 19:00:22 -0600247 dirname = options.branch.replace('/', '_')
Simon Glasse87bde12014-12-01 17:33:55 -0700248 # As a special case allow the board directory to be placed in the
249 # output directory itself rather than any subdirectory.
250 if not options.no_subdirs:
251 output_dir = os.path.join(options.output_dir, dirname)
Simon Glass30a664d2014-12-01 17:33:56 -0700252 if (clean_dir and output_dir != options.output_dir and
253 os.path.exists(output_dir)):
Simon Glassa10ebe12014-09-05 19:00:18 -0600254 shutil.rmtree(output_dir)
Simon Glassc05694f2013-04-03 11:07:16 +0000255 builder = Builder(toolchains, output_dir, options.git_dir,
Masahiro Yamada1fe610d2014-07-22 11:19:09 +0900256 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
Simon Glasse87bde12014-12-01 17:33:55 -0700257 show_unknown=options.show_unknown, step=options.step,
Simon Glass655b6102014-12-01 17:34:07 -0700258 no_subdirs=options.no_subdirs, full_path=options.full_path,
Stephen Warren97c96902016-04-11 10:48:44 -0600259 verbose_build=options.verbose_build,
260 incremental=options.incremental,
261 per_board_out_dir=options.per_board_out_dir,)
Simon Glassc05694f2013-04-03 11:07:16 +0000262 builder.force_config_on_failure = not options.quick
Simon Glassed098bb2014-09-05 19:00:13 -0600263 if make_func:
264 builder.do_make = make_func
Simon Glassc05694f2013-04-03 11:07:16 +0000265
266 # For a dry run, just show our actions as a sanity check
267 if options.dry_run:
268 ShowActions(series, why_selected, selected, builder, options)
269 else:
270 builder.force_build = options.force_build
Simon Glass7041c392014-07-13 12:22:31 -0600271 builder.force_build_failures = options.force_build_failures
Simon Glassf3018b7a2014-07-14 17:51:02 -0600272 builder.force_reconfig = options.force_reconfig
Simon Glass38df2e22014-07-14 17:51:03 -0600273 builder.in_tree = options.in_tree
Simon Glassc05694f2013-04-03 11:07:16 +0000274
275 # Work out which boards to build
276 board_selected = boards.GetSelectedDict()
277
Simon Glassd326ad72014-08-09 15:32:59 -0600278 if series:
279 commits = series.commits
Simon Glassa10ebe12014-09-05 19:00:18 -0600280 # Number the commits for test purposes
281 for commit in range(len(commits)):
282 commits[commit].sequence = commit
Simon Glassd326ad72014-08-09 15:32:59 -0600283 else:
284 commits = None
285
Simon Glassed098bb2014-09-05 19:00:13 -0600286 Print(GetActionSummary(options.summary, commits, board_selected,
287 options))
Simon Glassc05694f2013-04-03 11:07:16 +0000288
Simon Glass232d8502014-09-14 20:23:16 -0600289 # We can't show function sizes without board details at present
290 if options.show_bloat:
291 options.show_detail = True
Simon Glasseb48bbc2014-08-09 15:33:02 -0600292 builder.SetDisplayOptions(options.show_errors, options.show_sizes,
Simon Glass3394c9f2014-08-28 09:43:43 -0600293 options.show_detail, options.show_bloat,
Simon Glassdb17fb82015-02-05 22:06:15 -0700294 options.list_error_boards,
295 options.show_config)
Simon Glassc05694f2013-04-03 11:07:16 +0000296 if options.summary:
Simon Glasseb48bbc2014-08-09 15:33:02 -0600297 builder.ShowSummary(commits, board_selected)
Simon Glassc05694f2013-04-03 11:07:16 +0000298 else:
Simon Glassc2f91072014-08-28 09:43:39 -0600299 fail, warned = builder.BuildBoards(commits, board_selected,
Simon Glass78e418e2014-08-09 15:33:03 -0600300 options.keep_outputs, options.verbose)
Simon Glassc2f91072014-08-28 09:43:39 -0600301 if fail:
302 return 128
303 elif warned:
304 return 129
305 return 0