blob: 6cbecc3d5c8035abc30d498b05270f800c68a5d7 [file] [log] [blame]
Simon Glass1f701862019-10-31 07:42:57 -06001#!/usr/bin/env python3
Tom Rini10e47792018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamadab6160812015-05-20 11:36:07 +09003#
4# Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5#
Masahiro Yamadab6160812015-05-20 11:36:07 +09006
7"""
8Move config options from headers to defconfig files.
9
Simon Glass83cc72e2021-07-21 21:35:51 -060010See doc/develop/moveconfig.rst for documentation.
Masahiro Yamadab6160812015-05-20 11:36:07 +090011"""
12
Simon Glassd9c1da22021-12-18 14:54:31 -070013from argparse import ArgumentParser
Markus Klotzbuecherbac0bb52020-02-12 20:46:44 +010014import asteval
Simon Glassc6e73cf2017-06-01 19:39:03 -060015import collections
Simon Glassb3464eb2021-12-18 14:54:35 -070016from contextlib import ExitStack
Masahiro Yamadaea8f5342016-07-25 19:15:24 +090017import copy
Masahiro Yamada573b3902016-07-25 19:15:25 +090018import difflib
Simon Glassbb57be72021-12-18 08:09:45 -070019import doctest
Masahiro Yamada0f6beda2016-05-19 15:52:07 +090020import filecmp
Masahiro Yamadab6160812015-05-20 11:36:07 +090021import fnmatch
Masahiro Yamada3984d6e2016-10-19 14:39:54 +090022import glob
Masahiro Yamadab6160812015-05-20 11:36:07 +090023import multiprocessing
Masahiro Yamadab6160812015-05-20 11:36:07 +090024import os
Simon Glass1f701862019-10-31 07:42:57 -060025import queue
Masahiro Yamadab6160812015-05-20 11:36:07 +090026import re
27import shutil
28import subprocess
29import sys
30import tempfile
Simon Glass43cf08f2017-06-01 19:39:02 -060031import threading
Masahiro Yamadab6160812015-05-20 11:36:07 +090032import time
Simon Glassbb57be72021-12-18 08:09:45 -070033import unittest
Masahiro Yamadab6160812015-05-20 11:36:07 +090034
Simon Glassf0d9c102020-04-17 18:09:02 -060035from buildman import bsettings
36from buildman import kconfiglib
37from buildman import toolchain
Simon Glass44116332017-06-15 21:39:33 -060038
Masahiro Yamadab6160812015-05-20 11:36:07 +090039SHOW_GNU_MAKE = 'scripts/show-gnu-make'
40SLEEP_TIME=0.03
41
Masahiro Yamadab6160812015-05-20 11:36:07 +090042STATE_IDLE = 0
43STATE_DEFCONFIG = 1
44STATE_AUTOCONF = 2
Joe Hershberger166edec2015-05-19 13:21:17 -050045STATE_SAVEDEFCONFIG = 3
Masahiro Yamadab6160812015-05-20 11:36:07 +090046
47ACTION_MOVE = 0
Masahiro Yamada5643d6e2016-05-19 15:51:56 +090048ACTION_NO_ENTRY = 1
Masahiro Yamada35204d92016-08-22 22:18:21 +090049ACTION_NO_ENTRY_WARN = 2
50ACTION_NO_CHANGE = 3
Masahiro Yamadab6160812015-05-20 11:36:07 +090051
52COLOR_BLACK = '0;30'
53COLOR_RED = '0;31'
54COLOR_GREEN = '0;32'
55COLOR_BROWN = '0;33'
56COLOR_BLUE = '0;34'
57COLOR_PURPLE = '0;35'
58COLOR_CYAN = '0;36'
59COLOR_LIGHT_GRAY = '0;37'
60COLOR_DARK_GRAY = '1;30'
61COLOR_LIGHT_RED = '1;31'
62COLOR_LIGHT_GREEN = '1;32'
63COLOR_YELLOW = '1;33'
64COLOR_LIGHT_BLUE = '1;34'
65COLOR_LIGHT_PURPLE = '1;35'
66COLOR_LIGHT_CYAN = '1;36'
67COLOR_WHITE = '1;37'
68
Simon Glass8fb5bd02017-06-01 19:39:01 -060069AUTO_CONF_PATH = 'include/config/auto.conf'
Simon Glass43cf08f2017-06-01 19:39:02 -060070CONFIG_DATABASE = 'moveconfig.db'
Simon Glass8fb5bd02017-06-01 19:39:01 -060071
Simon Glass44116332017-06-15 21:39:33 -060072CONFIG_LEN = len('CONFIG_')
Simon Glass8fb5bd02017-06-01 19:39:01 -060073
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +020074SIZES = {
Simon Glassdc634d92021-12-18 14:54:30 -070075 'SZ_1': 0x00000001, 'SZ_2': 0x00000002,
76 'SZ_4': 0x00000004, 'SZ_8': 0x00000008,
77 'SZ_16': 0x00000010, 'SZ_32': 0x00000020,
78 'SZ_64': 0x00000040, 'SZ_128': 0x00000080,
79 'SZ_256': 0x00000100, 'SZ_512': 0x00000200,
80 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800,
81 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000,
82 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000,
83 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000,
84 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000,
85 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000,
86 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000,
87 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000,
88 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000,
89 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000,
90 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000,
91 'SZ_4G': 0x100000000
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +020092}
93
Simon Glasse8037552022-02-08 11:49:45 -070094RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig')
95
Simon Glass4c4eb7c2023-02-01 13:19:12 -070096# CONFIG symbols present in the build system (from Linux) but not actually used
97# in U-Boot; KCONFIG symbols
98IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL',
99 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL',
100 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT',
101 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3',
102 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO',
103 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING',
104 'VAL', '_UNDEFINED', 'SPL_BUILD', ]
105
106SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_']
107
Masahiro Yamadab6160812015-05-20 11:36:07 +0900108### helper functions ###
Masahiro Yamadab6160812015-05-20 11:36:07 +0900109def check_top_directory():
110 """Exit if we are not at the top of source directory."""
Simon Glassb3464eb2021-12-18 14:54:35 -0700111 for fname in 'README', 'Licenses':
112 if not os.path.exists(fname):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900113 sys.exit('Please run at the top of source directory.')
114
Masahiro Yamada990e6772016-05-19 15:51:54 +0900115def check_clean_directory():
116 """Exit if the source tree is not clean."""
Simon Glassb3464eb2021-12-18 14:54:35 -0700117 for fname in '.config', 'include/config':
118 if os.path.exists(fname):
Masahiro Yamada990e6772016-05-19 15:51:54 +0900119 sys.exit("source tree is not clean, please run 'make mrproper'")
120
Masahiro Yamadab6160812015-05-20 11:36:07 +0900121def get_make_cmd():
122 """Get the command name of GNU Make.
123
124 U-Boot needs GNU Make for building, but the command name is not
125 necessarily "make". (for example, "gmake" on FreeBSD).
126 Returns the most appropriate command name on your system.
127 """
Simon Glassb3464eb2021-12-18 14:54:35 -0700128 with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc:
129 ret = proc.communicate()
130 if proc.returncode:
131 sys.exit('GNU Make not found')
Masahiro Yamadab6160812015-05-20 11:36:07 +0900132 return ret[0].rstrip()
133
Simon Glass18774bc2017-06-01 19:38:58 -0600134def get_matched_defconfig(line):
135 """Get the defconfig files that match a pattern
136
137 Args:
Simon Glassb3464eb2021-12-18 14:54:35 -0700138 line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or
Simon Glass18774bc2017-06-01 19:38:58 -0600139 'k2*_defconfig'. If no directory is provided, 'configs/' is
140 prepended
141
142 Returns:
Simon Glassb3464eb2021-12-18 14:54:35 -0700143 list of str: a list of matching defconfig files
Simon Glass18774bc2017-06-01 19:38:58 -0600144 """
145 dirname = os.path.dirname(line)
146 if dirname:
147 pattern = line
148 else:
149 pattern = os.path.join('configs', line)
150 return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
151
Masahiro Yamada3984d6e2016-10-19 14:39:54 +0900152def get_matched_defconfigs(defconfigs_file):
Simon Glass8f3cf312017-06-01 19:38:59 -0600153 """Get all the defconfig files that match the patterns in a file.
154
155 Args:
Simon Glassb3464eb2021-12-18 14:54:35 -0700156 defconfigs_file (str): File containing a list of defconfigs to process,
157 or '-' to read the list from stdin
Simon Glass8f3cf312017-06-01 19:38:59 -0600158
159 Returns:
Simon Glassb3464eb2021-12-18 14:54:35 -0700160 list of str: A list of paths to defconfig files, with no duplicates
Simon Glass8f3cf312017-06-01 19:38:59 -0600161 """
Masahiro Yamada3984d6e2016-10-19 14:39:54 +0900162 defconfigs = []
Simon Glassb3464eb2021-12-18 14:54:35 -0700163 with ExitStack() as stack:
164 if defconfigs_file == '-':
165 inf = sys.stdin
166 defconfigs_file = 'stdin'
167 else:
168 inf = stack.enter_context(open(defconfigs_file, encoding='utf-8'))
169 for i, line in enumerate(inf):
170 line = line.strip()
171 if not line:
172 continue # skip blank lines silently
173 if ' ' in line:
174 line = line.split(' ')[0] # handle 'git log' input
175 matched = get_matched_defconfig(line)
176 if not matched:
177 print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'",
178 file=sys.stderr)
Masahiro Yamada3984d6e2016-10-19 14:39:54 +0900179
Simon Glassb3464eb2021-12-18 14:54:35 -0700180 defconfigs += matched
Masahiro Yamada3984d6e2016-10-19 14:39:54 +0900181
182 # use set() to drop multiple matching
Simon Glassb3464eb2021-12-18 14:54:35 -0700183 return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)]
Masahiro Yamada3984d6e2016-10-19 14:39:54 +0900184
Masahiro Yamada58175e32016-07-25 19:15:28 +0900185def get_all_defconfigs():
Simon Glassb3464eb2021-12-18 14:54:35 -0700186 """Get all the defconfig files under the configs/ directory.
187
188 Returns:
189 list of str: List of paths to defconfig files
190 """
Masahiro Yamada58175e32016-07-25 19:15:28 +0900191 defconfigs = []
Simon Glassb3464eb2021-12-18 14:54:35 -0700192 for (dirpath, _, filenames) in os.walk('configs'):
Masahiro Yamada58175e32016-07-25 19:15:28 +0900193 dirpath = dirpath[len('configs') + 1:]
194 for filename in fnmatch.filter(filenames, '*_defconfig'):
195 defconfigs.append(os.path.join(dirpath, filename))
196
197 return defconfigs
198
Masahiro Yamadab6160812015-05-20 11:36:07 +0900199def color_text(color_enabled, color, string):
200 """Return colored string."""
201 if color_enabled:
Masahiro Yamada465b7c02016-05-19 15:52:02 +0900202 # LF should not be surrounded by the escape sequence.
203 # Otherwise, additional whitespace or line-feed might be printed.
204 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
205 for s in string.split('\n') ])
Simon Glassb3464eb2021-12-18 14:54:35 -0700206 return string
Masahiro Yamadab6160812015-05-20 11:36:07 +0900207
Simon Glassb3464eb2021-12-18 14:54:35 -0700208def show_diff(alines, blines, file_path, color_enabled):
Masahiro Yamada573b3902016-07-25 19:15:25 +0900209 """Show unidified diff.
210
Simon Glassb3464eb2021-12-18 14:54:35 -0700211 Args:
212 alines (list of str): A list of lines (before)
213 blines (list of str): A list of lines (after)
214 file_path (str): Path to the file
215 color_enabled (bool): Display the diff in color
Masahiro Yamada573b3902016-07-25 19:15:25 +0900216 """
Simon Glassb3464eb2021-12-18 14:54:35 -0700217 diff = difflib.unified_diff(alines, blines,
Masahiro Yamada573b3902016-07-25 19:15:25 +0900218 fromfile=os.path.join('a', file_path),
219 tofile=os.path.join('b', file_path))
220
221 for line in diff:
Alper Nebi Yasakcab84072022-01-29 18:22:08 +0300222 if line.startswith('-') and not line.startswith('--'):
223 print(color_text(color_enabled, COLOR_RED, line))
224 elif line.startswith('+') and not line.startswith('++'):
225 print(color_text(color_enabled, COLOR_GREEN, line))
Masahiro Yamadaa1a4b092016-07-25 19:15:26 +0900226 else:
Alper Nebi Yasakcab84072022-01-29 18:22:08 +0300227 print(line)
Masahiro Yamada573b3902016-07-25 19:15:25 +0900228
Simon Glassb3464eb2021-12-18 14:54:35 -0700229def extend_matched_lines(lines, matched, pre_patterns, post_patterns,
230 extend_pre, extend_post):
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900231 """Extend matched lines if desired patterns are found before/after already
232 matched lines.
233
Simon Glassb3464eb2021-12-18 14:54:35 -0700234 Args:
235 lines (list of str): list of lines handled.
236 matched (list of int): list of line numbers that have been already
237 matched (will be updated by this function)
238 pre_patterns (list of re.Pattern): list of regular expression that should
239 be matched as preamble
240 post_patterns (list of re.Pattern): list of regular expression that should
241 be matched as postamble
242 extend_pre (bool): Add the line number of matched preamble to the matched
243 list
244 extend_post (bool): Add the line number of matched postamble to the
245 matched list
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900246 """
247 extended_matched = []
248
249 j = matched[0]
250
251 for i in matched:
252 if i == 0 or i < j:
253 continue
254 j = i
255 while j in matched:
256 j += 1
257 if j >= len(lines):
258 break
259
Simon Glassb3464eb2021-12-18 14:54:35 -0700260 for pat in pre_patterns:
261 if pat.search(lines[i - 1]):
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900262 break
263 else:
264 # not matched
265 continue
266
Simon Glassb3464eb2021-12-18 14:54:35 -0700267 for pat in post_patterns:
268 if pat.search(lines[j]):
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900269 break
270 else:
271 # not matched
272 continue
273
274 if extend_pre:
275 extended_matched.append(i - 1)
276 if extend_post:
277 extended_matched.append(j)
278
279 matched += extended_matched
280 matched.sort()
281
Simon Glassd9c1da22021-12-18 14:54:31 -0700282def confirm(args, prompt):
Simon Glassb3464eb2021-12-18 14:54:35 -0700283 """Ask the user to confirm something
284
285 Args:
286 args (Namespace ): program arguments
287
288 Returns:
289 bool: True to confirm, False to cancel/stop
290 """
Simon Glassd9c1da22021-12-18 14:54:31 -0700291 if not args.yes:
Chris Packham85e15c52017-05-02 21:30:46 +1200292 while True:
Simon Glassb3464eb2021-12-18 14:54:35 -0700293 choice = input(f'{prompt} [y/n]: ')
Chris Packham85e15c52017-05-02 21:30:46 +1200294 choice = choice.lower()
Simon Glass1f701862019-10-31 07:42:57 -0600295 print(choice)
Simon Glassb3464eb2021-12-18 14:54:35 -0700296 if choice in ('y', 'n'):
Chris Packham85e15c52017-05-02 21:30:46 +1200297 break
298
299 if choice == 'n':
300 return False
301
302 return True
303
Simon Glassb09ae452021-12-18 14:54:33 -0700304def write_file(fname, data):
305 """Write data to a file
306
307 Args:
308 fname (str): Filename to write to
309 data (list of str): Lines to write (with or without trailing newline);
310 or str to write
311 """
312 with open(fname, 'w', encoding='utf-8') as out:
313 if isinstance(data, list):
314 for line in data:
315 print(line.rstrip('\n'), file=out)
316 else:
317 out.write(data)
318
Simon Glassaba238f2021-12-18 14:54:34 -0700319def read_file(fname, as_lines=True, skip_unicode=False):
320 """Read a file and return the contents
321
322 Args:
323 fname (str): Filename to read from
324 as_lines: Return file contents as a list of lines
325 skip_unicode (bool): True to report unicode errors and continue
326
327 Returns:
328 iter of str: List of ;ines from the file with newline removed; str if
329 as_lines is False with newlines intact; or None if a unicode error
330 occurred
331
332 Raises:
333 UnicodeDecodeError: Unicode error occurred when reading
334 """
335 with open(fname, encoding='utf-8') as inf:
336 try:
337 if as_lines:
338 return [line.rstrip('\n') for line in inf.readlines()]
339 else:
340 return inf.read()
341 except UnicodeDecodeError as e:
342 if not skip_unicode:
Simon Glassafaddc72022-02-11 13:23:22 -0700343 raise
Simon Glassaba238f2021-12-18 14:54:34 -0700344 print("Failed on file %s': %s" % (fname, e))
345 return None
346
Simon Glassd9c1da22021-12-18 14:54:31 -0700347def cleanup_empty_blocks(header_path, args):
Chris Packham73f6c7c2019-01-30 20:23:16 +1300348 """Clean up empty conditional blocks
349
Simon Glassb3464eb2021-12-18 14:54:35 -0700350 Args:
351 header_path (str): path to the cleaned file.
352 args (Namespace): program arguments
Chris Packham73f6c7c2019-01-30 20:23:16 +1300353 """
354 pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M)
Simon Glassaba238f2021-12-18 14:54:34 -0700355 data = read_file(header_path, as_lines=False, skip_unicode=True)
356 if data is None:
357 return
Chris Packham73f6c7c2019-01-30 20:23:16 +1300358
359 new_data = pattern.sub('\n', data)
360
361 show_diff(data.splitlines(True), new_data.splitlines(True), header_path,
Simon Glassd9c1da22021-12-18 14:54:31 -0700362 args.color)
Chris Packham73f6c7c2019-01-30 20:23:16 +1300363
Simon Glassd9c1da22021-12-18 14:54:31 -0700364 if args.dry_run:
Chris Packham73f6c7c2019-01-30 20:23:16 +1300365 return
366
Simon Glassaba238f2021-12-18 14:54:34 -0700367 if new_data != data:
368 write_file(header_path, new_data)
Chris Packham73f6c7c2019-01-30 20:23:16 +1300369
Simon Glassd9c1da22021-12-18 14:54:31 -0700370def cleanup_one_header(header_path, patterns, args):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900371 """Clean regex-matched lines away from a file.
372
Simon Glassb3464eb2021-12-18 14:54:35 -0700373 Args:
Masahiro Yamadab6160812015-05-20 11:36:07 +0900374 header_path: path to the cleaned file.
375 patterns: list of regex patterns. Any lines matching to these
376 patterns are deleted.
Simon Glassb3464eb2021-12-18 14:54:35 -0700377 args (Namespace): program arguments
Masahiro Yamadab6160812015-05-20 11:36:07 +0900378 """
Simon Glassaba238f2021-12-18 14:54:34 -0700379 lines = read_file(header_path, skip_unicode=True)
380 if lines is None:
381 return
Masahiro Yamadab6160812015-05-20 11:36:07 +0900382
383 matched = []
384 for i, line in enumerate(lines):
Alper Nebi Yasakcab84072022-01-29 18:22:08 +0300385 if i - 1 in matched and lines[i - 1].endswith('\\'):
Masahiro Yamada6d798ba2016-07-25 19:15:27 +0900386 matched.append(i)
387 continue
Masahiro Yamadab6160812015-05-20 11:36:07 +0900388 for pattern in patterns:
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900389 if pattern.search(line):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900390 matched.append(i)
391 break
392
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900393 if not matched:
394 return
395
396 # remove empty #ifdef ... #endif, successive blank lines
Alper Nebi Yasakcab84072022-01-29 18:22:08 +0300397 pattern_if = re.compile(r'#\s*if(def|ndef)?\b') # #if, #ifdef, #ifndef
398 pattern_elif = re.compile(r'#\s*el(if|se)\b') # #elif, #else
399 pattern_endif = re.compile(r'#\s*endif\b') # #endif
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900400 pattern_blank = re.compile(r'^\s*$') # empty line
401
402 while True:
403 old_matched = copy.copy(matched)
404 extend_matched_lines(lines, matched, [pattern_if],
405 [pattern_endif], True, True)
406 extend_matched_lines(lines, matched, [pattern_elif],
407 [pattern_elif, pattern_endif], True, False)
408 extend_matched_lines(lines, matched, [pattern_if, pattern_elif],
409 [pattern_blank], False, True)
410 extend_matched_lines(lines, matched, [pattern_blank],
411 [pattern_elif, pattern_endif], True, False)
412 extend_matched_lines(lines, matched, [pattern_blank],
413 [pattern_blank], True, False)
414 if matched == old_matched:
415 break
416
Masahiro Yamada573b3902016-07-25 19:15:25 +0900417 tolines = copy.copy(lines)
418
419 for i in reversed(matched):
420 tolines.pop(i)
421
Simon Glassd9c1da22021-12-18 14:54:31 -0700422 show_diff(lines, tolines, header_path, args.color)
Masahiro Yamadaea8f5342016-07-25 19:15:24 +0900423
Simon Glassd9c1da22021-12-18 14:54:31 -0700424 if args.dry_run:
Masahiro Yamadab6160812015-05-20 11:36:07 +0900425 return
426
Simon Glassb09ae452021-12-18 14:54:33 -0700427 write_file(header_path, tolines)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900428
Simon Glassd9c1da22021-12-18 14:54:31 -0700429def cleanup_headers(configs, args):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900430 """Delete config defines from board headers.
431
Simon Glassb3464eb2021-12-18 14:54:35 -0700432 Args:
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900433 configs: A list of CONFIGs to remove.
Simon Glassb3464eb2021-12-18 14:54:35 -0700434 args (Namespace): program arguments
Masahiro Yamadab6160812015-05-20 11:36:07 +0900435 """
Simon Glassd9c1da22021-12-18 14:54:31 -0700436 if not confirm(args, 'Clean up headers?'):
Chris Packham85e15c52017-05-02 21:30:46 +1200437 return
Masahiro Yamadab6160812015-05-20 11:36:07 +0900438
439 patterns = []
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900440 for config in configs:
Alper Nebi Yasakcab84072022-01-29 18:22:08 +0300441 patterns.append(re.compile(r'#\s*define\s+%s\b' % config))
442 patterns.append(re.compile(r'#\s*undef\s+%s\b' % config))
Masahiro Yamadab6160812015-05-20 11:36:07 +0900443
Joe Hershbergerb78ad422015-05-19 13:21:21 -0500444 for dir in 'include', 'arch', 'board':
445 for (dirpath, dirnames, filenames) in os.walk(dir):
Masahiro Yamada28a6d352016-07-25 19:15:22 +0900446 if dirpath == os.path.join('include', 'generated'):
447 continue
Joe Hershbergerb78ad422015-05-19 13:21:21 -0500448 for filename in filenames:
Simon Glassce44bf52020-08-11 11:23:34 -0600449 if not filename.endswith(('~', '.dts', '.dtsi', '.bin',
Trevor Woerneraaad0c22021-03-15 12:01:33 -0400450 '.elf','.aml','.dat')):
Chris Packham73f6c7c2019-01-30 20:23:16 +1300451 header_path = os.path.join(dirpath, filename)
Tom Rinie90c0072019-11-10 21:19:37 -0500452 # This file contains UTF-16 data and no CONFIG symbols
453 if header_path == 'include/video_font_data.h':
454 continue
Simon Glassd9c1da22021-12-18 14:54:31 -0700455 cleanup_one_header(header_path, patterns, args)
456 cleanup_empty_blocks(header_path, args)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900457
Chris Packham0e6deff2017-05-02 21:30:48 +1200458def find_matching(patterns, line):
459 for pat in patterns:
460 if pat.search(line):
461 return True
462 return False
463
Simon Glassd9c1da22021-12-18 14:54:31 -0700464def cleanup_readme(configs, args):
Chris Packham0e6deff2017-05-02 21:30:48 +1200465 """Delete config description in README
466
Simon Glassb3464eb2021-12-18 14:54:35 -0700467 Args:
Chris Packham0e6deff2017-05-02 21:30:48 +1200468 configs: A list of CONFIGs to remove.
Simon Glassb3464eb2021-12-18 14:54:35 -0700469 args (Namespace): program arguments
Chris Packham0e6deff2017-05-02 21:30:48 +1200470 """
Simon Glassd9c1da22021-12-18 14:54:31 -0700471 if not confirm(args, 'Clean up README?'):
Chris Packham0e6deff2017-05-02 21:30:48 +1200472 return
473
474 patterns = []
475 for config in configs:
476 patterns.append(re.compile(r'^\s+%s' % config))
477
Simon Glassaba238f2021-12-18 14:54:34 -0700478 lines = read_file('README')
Chris Packham0e6deff2017-05-02 21:30:48 +1200479
480 found = False
481 newlines = []
482 for line in lines:
483 if not found:
484 found = find_matching(patterns, line)
485 if found:
486 continue
487
488 if found and re.search(r'^\s+CONFIG', line):
489 found = False
490
491 if not found:
492 newlines.append(line)
493
Simon Glassb09ae452021-12-18 14:54:33 -0700494 write_file('README', newlines)
Chris Packham0e6deff2017-05-02 21:30:48 +1200495
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +0200496def try_expand(line):
497 """If value looks like an expression, try expanding it
498 Otherwise just return the existing value
499 """
500 if line.find('=') == -1:
501 return line
502
503 try:
Markus Klotzbuecherbac0bb52020-02-12 20:46:44 +0100504 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +0200505 cfg, val = re.split("=", line)
506 val= val.strip('\"')
Simon Glassdc634d92021-12-18 14:54:30 -0700507 if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val):
Markus Klotzbuecherbac0bb52020-02-12 20:46:44 +0100508 newval = hex(aeval(val))
Simon Glassdc634d92021-12-18 14:54:30 -0700509 print('\tExpanded expression %s to %s' % (val, newval))
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +0200510 return cfg+'='+newval
511 except:
Simon Glassdc634d92021-12-18 14:54:30 -0700512 print('\tFailed to expand expression in %s' % line)
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +0200513
514 return line
515
Chris Packham9d5274f2017-05-02 21:30:47 +1200516
Masahiro Yamadab6160812015-05-20 11:36:07 +0900517### classes ###
Masahiro Yamadacefaa582016-05-19 15:51:55 +0900518class Progress:
519
520 """Progress Indicator"""
521
522 def __init__(self, total):
523 """Create a new progress indicator.
524
Simon Glassb3464eb2021-12-18 14:54:35 -0700525 Args:
Masahiro Yamadacefaa582016-05-19 15:51:55 +0900526 total: A number of defconfig files to process.
527 """
528 self.current = 0
529 self.total = total
530
531 def inc(self):
532 """Increment the number of processed defconfig files."""
533
534 self.current += 1
535
536 def show(self):
537 """Display the progress."""
Simon Glass1f701862019-10-31 07:42:57 -0600538 print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ')
Masahiro Yamadacefaa582016-05-19 15:51:55 +0900539 sys.stdout.flush()
540
Simon Glass44116332017-06-15 21:39:33 -0600541
542class KconfigScanner:
543 """Kconfig scanner."""
544
545 def __init__(self):
546 """Scan all the Kconfig files and create a Config object."""
547 # Define environment variables referenced from Kconfig
548 os.environ['srctree'] = os.getcwd()
549 os.environ['UBOOTVERSION'] = 'dummy'
550 os.environ['KCONFIG_OBJDIR'] = ''
Simon Glass4c4eb7c2023-02-01 13:19:12 -0700551 os.environ['CC'] = 'gcc'
Tom Rini3c5f4152019-09-20 17:42:09 -0400552 self.conf = kconfiglib.Kconfig()
Simon Glass44116332017-06-15 21:39:33 -0600553
554
Masahiro Yamadab6160812015-05-20 11:36:07 +0900555class KconfigParser:
556
557 """A parser of .config and include/autoconf.mk."""
558
559 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
560 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
561
Simon Glassd9c1da22021-12-18 14:54:31 -0700562 def __init__(self, configs, args, build_dir):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900563 """Create a new parser.
564
Simon Glassb3464eb2021-12-18 14:54:35 -0700565 Args:
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900566 configs: A list of CONFIGs to move.
Simon Glassb3464eb2021-12-18 14:54:35 -0700567 args (Namespace): program arguments
Masahiro Yamadab6160812015-05-20 11:36:07 +0900568 build_dir: Build directory.
569 """
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900570 self.configs = configs
Simon Glassd9c1da22021-12-18 14:54:31 -0700571 self.args = args
Masahiro Yamada5393b612016-05-19 15:52:00 +0900572 self.dotconfig = os.path.join(build_dir, '.config')
573 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
Masahiro Yamada6d139172016-08-22 22:18:22 +0900574 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
575 'autoconf.mk')
Simon Glass8fb5bd02017-06-01 19:39:01 -0600576 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
Masahiro Yamada07f98522016-05-19 15:52:06 +0900577 self.defconfig = os.path.join(build_dir, 'defconfig')
Masahiro Yamadab6160812015-05-20 11:36:07 +0900578
Simon Glass257f5232017-07-10 14:47:47 -0600579 def get_arch(self):
580 """Parse .config file and return the architecture.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900581
582 Returns:
Simon Glass257f5232017-07-10 14:47:47 -0600583 Architecture name (e.g. 'arm').
Masahiro Yamadab6160812015-05-20 11:36:07 +0900584 """
585 arch = ''
586 cpu = ''
Simon Glassaba238f2021-12-18 14:54:34 -0700587 for line in read_file(self.dotconfig):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900588 m = self.re_arch.match(line)
589 if m:
590 arch = m.group(1)
591 continue
592 m = self.re_cpu.match(line)
593 if m:
594 cpu = m.group(1)
595
Masahiro Yamadac4d76eb2016-05-19 15:51:53 +0900596 if not arch:
597 return None
Masahiro Yamadab6160812015-05-20 11:36:07 +0900598
599 # fix-up for aarch64
600 if arch == 'arm' and cpu == 'armv8':
601 arch = 'aarch64'
602
Simon Glass257f5232017-07-10 14:47:47 -0600603 return arch
Masahiro Yamadab6160812015-05-20 11:36:07 +0900604
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900605 def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900606 """Parse .config, defconfig, include/autoconf.mk for one config.
607
608 This function looks for the config options in the lines from
609 defconfig, .config, and include/autoconf.mk in order to decide
610 which action should be taken for this defconfig.
611
Simon Glassb3464eb2021-12-18 14:54:35 -0700612 Args:
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900613 config: CONFIG name to parse.
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900614 dotconfig_lines: lines from the .config file.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900615 autoconf_lines: lines from the include/autoconf.mk file.
616
617 Returns:
618 A tupple of the action for this defconfig and the line
619 matched for the config.
620 """
Masahiro Yamadab6160812015-05-20 11:36:07 +0900621 not_set = '# %s is not set' % config
622
Masahiro Yamadab6160812015-05-20 11:36:07 +0900623 for line in autoconf_lines:
624 line = line.rstrip()
625 if line.startswith(config + '='):
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900626 new_val = line
Masahiro Yamadab6160812015-05-20 11:36:07 +0900627 break
Masahiro Yamadab6160812015-05-20 11:36:07 +0900628 else:
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900629 new_val = not_set
630
Markus Klotzbuecher3d5d4182019-05-15 15:15:52 +0200631 new_val = try_expand(new_val)
632
Masahiro Yamada35204d92016-08-22 22:18:21 +0900633 for line in dotconfig_lines:
634 line = line.rstrip()
635 if line.startswith(config + '=') or line == not_set:
636 old_val = line
637 break
638 else:
639 if new_val == not_set:
640 return (ACTION_NO_ENTRY, config)
641 else:
642 return (ACTION_NO_ENTRY_WARN, config)
643
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900644 # If this CONFIG is neither bool nor trisate
645 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
646 # tools/scripts/define2mk.sed changes '1' to 'y'.
647 # This is a problem if the CONFIG is int type.
648 # Check the type in Kconfig and handle it correctly.
649 if new_val[-2:] == '=y':
650 new_val = new_val[:-1] + '1'
651
Masahiro Yamadab48387f2016-06-15 14:33:50 +0900652 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE,
653 new_val)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900654
Masahiro Yamada465b7c02016-05-19 15:52:02 +0900655 def update_dotconfig(self):
Masahiro Yamada7c0d9d22016-05-19 15:51:50 +0900656 """Parse files for the config options and update the .config.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900657
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900658 This function parses the generated .config and include/autoconf.mk
659 searching the target options.
Masahiro Yamada7c0d9d22016-05-19 15:51:50 +0900660 Move the config option(s) to the .config as needed.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900661
Simon Glassb3464eb2021-12-18 14:54:35 -0700662 Args:
Masahiro Yamadab6160812015-05-20 11:36:07 +0900663 defconfig: defconfig name.
Masahiro Yamada69e2bbc2016-05-19 15:52:01 +0900664
665 Returns:
Masahiro Yamada263d1372016-05-19 15:52:04 +0900666 Return a tuple of (updated flag, log string).
667 The "updated flag" is True if the .config was updated, False
668 otherwise. The "log string" shows what happend to the .config.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900669 """
670
Masahiro Yamadab6160812015-05-20 11:36:07 +0900671 results = []
Masahiro Yamada263d1372016-05-19 15:52:04 +0900672 updated = False
Masahiro Yamada35204d92016-08-22 22:18:21 +0900673 suspicious = False
Masahiro Yamada6d139172016-08-22 22:18:22 +0900674 rm_files = [self.config_autoconf, self.autoconf]
675
Simon Glassd9c1da22021-12-18 14:54:31 -0700676 if self.args.spl:
Masahiro Yamada6d139172016-08-22 22:18:22 +0900677 if os.path.exists(self.spl_autoconf):
678 autoconf_path = self.spl_autoconf
679 rm_files.append(self.spl_autoconf)
680 else:
681 for f in rm_files:
682 os.remove(f)
683 return (updated, suspicious,
Simon Glassd9c1da22021-12-18 14:54:31 -0700684 color_text(self.args.color, COLOR_BROWN,
Masahiro Yamada6d139172016-08-22 22:18:22 +0900685 "SPL is not enabled. Skipped.") + '\n')
686 else:
687 autoconf_path = self.autoconf
Masahiro Yamadab6160812015-05-20 11:36:07 +0900688
Simon Glassaba238f2021-12-18 14:54:34 -0700689 dotconfig_lines = read_file(self.dotconfig)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900690
Simon Glassaba238f2021-12-18 14:54:34 -0700691 autoconf_lines = read_file(autoconf_path)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900692
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900693 for config in self.configs:
694 result = self.parse_one_config(config, dotconfig_lines,
Joe Hershberger166edec2015-05-19 13:21:17 -0500695 autoconf_lines)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900696 results.append(result)
697
698 log = ''
699
700 for (action, value) in results:
701 if action == ACTION_MOVE:
702 actlog = "Move '%s'" % value
703 log_color = COLOR_LIGHT_GREEN
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900704 elif action == ACTION_NO_ENTRY:
Simon Glassdc634d92021-12-18 14:54:30 -0700705 actlog = '%s is not defined in Kconfig. Do nothing.' % value
Masahiro Yamadab6160812015-05-20 11:36:07 +0900706 log_color = COLOR_LIGHT_BLUE
Masahiro Yamada35204d92016-08-22 22:18:21 +0900707 elif action == ACTION_NO_ENTRY_WARN:
Simon Glassdc634d92021-12-18 14:54:30 -0700708 actlog = '%s is not defined in Kconfig (suspicious). Do nothing.' % value
Masahiro Yamada35204d92016-08-22 22:18:21 +0900709 log_color = COLOR_YELLOW
710 suspicious = True
Masahiro Yamada5643d6e2016-05-19 15:51:56 +0900711 elif action == ACTION_NO_CHANGE:
712 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \
713 % value
Masahiro Yamadab6160812015-05-20 11:36:07 +0900714 log_color = COLOR_LIGHT_PURPLE
Masahiro Yamadab6160812015-05-20 11:36:07 +0900715 else:
Simon Glassdc634d92021-12-18 14:54:30 -0700716 sys.exit('Internal Error. This should not happen.')
Masahiro Yamadab6160812015-05-20 11:36:07 +0900717
Simon Glassd9c1da22021-12-18 14:54:31 -0700718 log += color_text(self.args.color, log_color, actlog) + '\n'
Masahiro Yamadab6160812015-05-20 11:36:07 +0900719
Simon Glassb3464eb2021-12-18 14:54:35 -0700720 with open(self.dotconfig, 'a', encoding='utf-8') as out:
Masahiro Yamada953d93b2016-05-19 15:51:49 +0900721 for (action, value) in results:
722 if action == ACTION_MOVE:
Simon Glassb3464eb2021-12-18 14:54:35 -0700723 out.write(value + '\n')
Masahiro Yamada263d1372016-05-19 15:52:04 +0900724 updated = True
Masahiro Yamadab6160812015-05-20 11:36:07 +0900725
Masahiro Yamada07f98522016-05-19 15:52:06 +0900726 self.results = results
Masahiro Yamada6d139172016-08-22 22:18:22 +0900727 for f in rm_files:
728 os.remove(f)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900729
Masahiro Yamada35204d92016-08-22 22:18:21 +0900730 return (updated, suspicious, log)
Masahiro Yamada69e2bbc2016-05-19 15:52:01 +0900731
Masahiro Yamada07f98522016-05-19 15:52:06 +0900732 def check_defconfig(self):
733 """Check the defconfig after savedefconfig
734
735 Returns:
736 Return additional log if moved CONFIGs were removed again by
737 'make savedefconfig'.
738 """
739
740 log = ''
741
Simon Glassaba238f2021-12-18 14:54:34 -0700742 defconfig_lines = read_file(self.defconfig)
Masahiro Yamada07f98522016-05-19 15:52:06 +0900743
744 for (action, value) in self.results:
745 if action != ACTION_MOVE:
746 continue
Alper Nebi Yasakcab84072022-01-29 18:22:08 +0300747 if not value in defconfig_lines:
Simon Glassd9c1da22021-12-18 14:54:31 -0700748 log += color_text(self.args.color, COLOR_YELLOW,
Masahiro Yamada07f98522016-05-19 15:52:06 +0900749 "'%s' was removed by savedefconfig.\n" %
750 value)
751
752 return log
753
Simon Glass43cf08f2017-06-01 19:39:02 -0600754
755class DatabaseThread(threading.Thread):
756 """This thread processes results from Slot threads.
757
758 It collects the data in the master config directary. There is only one
759 result thread, and this helps to serialise the build output.
760 """
761 def __init__(self, config_db, db_queue):
762 """Set up a new result thread
763
764 Args:
765 builder: Builder which will be sent each result
766 """
767 threading.Thread.__init__(self)
768 self.config_db = config_db
769 self.db_queue= db_queue
770
771 def run(self):
772 """Called to start up the result thread.
773
774 We collect the next result job and pass it on to the build.
775 """
776 while True:
777 defconfig, configs = self.db_queue.get()
778 self.config_db[defconfig] = configs
779 self.db_queue.task_done()
780
781
Masahiro Yamadab6160812015-05-20 11:36:07 +0900782class Slot:
783
784 """A slot to store a subprocess.
785
786 Each instance of this class handles one subprocess.
787 This class is useful to control multiple threads
788 for faster processing.
789 """
790
Simon Glassd9c1da22021-12-18 14:54:31 -0700791 def __init__(self, toolchains, configs, args, progress, devnull,
Simon Glass257f5232017-07-10 14:47:47 -0600792 make_cmd, reference_src_dir, db_queue):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900793 """Create a new process slot.
794
Simon Glassb3464eb2021-12-18 14:54:35 -0700795 Args:
Simon Glass257f5232017-07-10 14:47:47 -0600796 toolchains: Toolchains object containing toolchains.
Masahiro Yamadab80a6672016-05-19 15:51:57 +0900797 configs: A list of CONFIGs to move.
Simon Glassd9c1da22021-12-18 14:54:31 -0700798 args: Program arguments
Masahiro Yamadacefaa582016-05-19 15:51:55 +0900799 progress: A progress indicator.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900800 devnull: A file object of '/dev/null'.
801 make_cmd: command name of GNU Make.
Joe Hershbergerb1a570f2016-06-10 14:53:32 -0500802 reference_src_dir: Determine the true starting config state from this
803 source tree.
Simon Glass43cf08f2017-06-01 19:39:02 -0600804 db_queue: output queue to write config info for the database
Masahiro Yamadab6160812015-05-20 11:36:07 +0900805 """
Simon Glass257f5232017-07-10 14:47:47 -0600806 self.toolchains = toolchains
Simon Glassd9c1da22021-12-18 14:54:31 -0700807 self.args = args
Masahiro Yamadacefaa582016-05-19 15:51:55 +0900808 self.progress = progress
Masahiro Yamadab6160812015-05-20 11:36:07 +0900809 self.build_dir = tempfile.mkdtemp()
810 self.devnull = devnull
811 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
Joe Hershbergerb1a570f2016-06-10 14:53:32 -0500812 self.reference_src_dir = reference_src_dir
Simon Glass43cf08f2017-06-01 19:39:02 -0600813 self.db_queue = db_queue
Simon Glassd9c1da22021-12-18 14:54:31 -0700814 self.parser = KconfigParser(configs, args, self.build_dir)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900815 self.state = STATE_IDLE
Masahiro Yamada1271b672016-08-22 22:18:20 +0900816 self.failed_boards = set()
817 self.suspicious_boards = set()
Masahiro Yamadab6160812015-05-20 11:36:07 +0900818
819 def __del__(self):
820 """Delete the working directory
821
822 This function makes sure the temporary directory is cleaned away
823 even if Python suddenly dies due to error. It should be done in here
Joe Hershberger640de872016-06-10 14:53:29 -0500824 because it is guaranteed the destructor is always invoked when the
Masahiro Yamadab6160812015-05-20 11:36:07 +0900825 instance of the class gets unreferenced.
826
827 If the subprocess is still running, wait until it finishes.
828 """
829 if self.state != STATE_IDLE:
830 while self.ps.poll() == None:
831 pass
832 shutil.rmtree(self.build_dir)
833
Masahiro Yamadacefaa582016-05-19 15:51:55 +0900834 def add(self, defconfig):
Masahiro Yamadab6160812015-05-20 11:36:07 +0900835 """Assign a new subprocess for defconfig and add it to the slot.
836
837 If the slot is vacant, create a new subprocess for processing the
838 given defconfig and add it to the slot. Just returns False if
839 the slot is occupied (i.e. the current subprocess is still running).
840
Simon Glassb3464eb2021-12-18 14:54:35 -0700841 Args:
Masahiro Yamadab6160812015-05-20 11:36:07 +0900842 defconfig: defconfig name.
843
844 Returns:
845 Return True on success or False on failure
846 """
847 if self.state != STATE_IDLE:
848 return False
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900849
Masahiro Yamadab6160812015-05-20 11:36:07 +0900850 self.defconfig = defconfig
Masahiro Yamada465b7c02016-05-19 15:52:02 +0900851 self.log = ''
Masahiro Yamada8f5256a2016-06-15 14:33:52 +0900852 self.current_src_dir = self.reference_src_dir
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900853 self.do_defconfig()
Masahiro Yamadab6160812015-05-20 11:36:07 +0900854 return True
855
856 def poll(self):
857 """Check the status of the subprocess and handle it as needed.
858
859 Returns True if the slot is vacant (i.e. in idle state).
860 If the configuration is successfully finished, assign a new
861 subprocess to build include/autoconf.mk.
862 If include/autoconf.mk is generated, invoke the parser to
Masahiro Yamada263d1372016-05-19 15:52:04 +0900863 parse the .config and the include/autoconf.mk, moving
864 config options to the .config as needed.
865 If the .config was updated, run "make savedefconfig" to sync
866 it, update the original defconfig, and then set the slot back
867 to the idle state.
Masahiro Yamadab6160812015-05-20 11:36:07 +0900868
869 Returns:
870 Return True if the subprocess is terminated, False otherwise
871 """
872 if self.state == STATE_IDLE:
873 return True
874
875 if self.ps.poll() == None:
876 return False
877
878 if self.ps.poll() != 0:
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900879 self.handle_error()
880 elif self.state == STATE_DEFCONFIG:
Masahiro Yamada8f5256a2016-06-15 14:33:52 +0900881 if self.reference_src_dir and not self.current_src_dir:
Joe Hershbergerb1a570f2016-06-10 14:53:32 -0500882 self.do_savedefconfig()
883 else:
884 self.do_autoconf()
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900885 elif self.state == STATE_AUTOCONF:
Masahiro Yamada8f5256a2016-06-15 14:33:52 +0900886 if self.current_src_dir:
887 self.current_src_dir = None
Joe Hershbergerb1a570f2016-06-10 14:53:32 -0500888 self.do_defconfig()
Simon Glassd9c1da22021-12-18 14:54:31 -0700889 elif self.args.build_db:
Simon Glass43cf08f2017-06-01 19:39:02 -0600890 self.do_build_db()
Joe Hershbergerb1a570f2016-06-10 14:53:32 -0500891 else:
892 self.do_savedefconfig()
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900893 elif self.state == STATE_SAVEDEFCONFIG:
894 self.update_defconfig()
895 else:
Simon Glassdc634d92021-12-18 14:54:30 -0700896 sys.exit('Internal Error. This should not happen.')
Masahiro Yamadab6160812015-05-20 11:36:07 +0900897
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900898 return True if self.state == STATE_IDLE else False
Joe Hershberger166edec2015-05-19 13:21:17 -0500899
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900900 def handle_error(self):
901 """Handle error cases."""
Masahiro Yamada83c17672016-05-19 15:52:08 +0900902
Simon Glassd9c1da22021-12-18 14:54:31 -0700903 self.log += color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glassdc634d92021-12-18 14:54:30 -0700904 'Failed to process.\n')
Simon Glassd9c1da22021-12-18 14:54:31 -0700905 if self.args.verbose:
906 self.log += color_text(self.args.color, COLOR_LIGHT_CYAN,
Markus Klotzbuecher8773f352020-02-12 20:46:45 +0100907 self.ps.stderr.read().decode())
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900908 self.finish(False)
Joe Hershberger166edec2015-05-19 13:21:17 -0500909
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900910 def do_defconfig(self):
911 """Run 'make <board>_defconfig' to create the .config file."""
Masahiro Yamada0f6beda2016-05-19 15:52:07 +0900912
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900913 cmd = list(self.make_cmd)
914 cmd.append(self.defconfig)
915 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
Masahiro Yamada8f5256a2016-06-15 14:33:52 +0900916 stderr=subprocess.PIPE,
917 cwd=self.current_src_dir)
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900918 self.state = STATE_DEFCONFIG
Masahiro Yamada0f6beda2016-05-19 15:52:07 +0900919
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900920 def do_autoconf(self):
Simon Glass8fb5bd02017-06-01 19:39:01 -0600921 """Run 'make AUTO_CONF_PATH'."""
Masahiro Yamadab6160812015-05-20 11:36:07 +0900922
Simon Glass257f5232017-07-10 14:47:47 -0600923 arch = self.parser.get_arch()
924 try:
925 toolchain = self.toolchains.Select(arch)
926 except ValueError:
Simon Glassd9c1da22021-12-18 14:54:31 -0700927 self.log += color_text(self.args.color, COLOR_YELLOW,
Chris Packham1ebcbd12017-08-27 20:00:51 +1200928 "Tool chain for '%s' is missing. Do nothing.\n" % arch)
Masahiro Yamada274a5ee2016-05-19 15:52:03 +0900929 self.finish(False)
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900930 return
Simon Glass1f701862019-10-31 07:42:57 -0600931 env = toolchain.MakeEnvironment(False)
Masahiro Yamadac4d76eb2016-05-19 15:51:53 +0900932
Masahiro Yamadab6160812015-05-20 11:36:07 +0900933 cmd = list(self.make_cmd)
Joe Hershberger765442b2015-05-19 13:21:18 -0500934 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
Simon Glass8fb5bd02017-06-01 19:39:01 -0600935 cmd.append(AUTO_CONF_PATH)
Simon Glass257f5232017-07-10 14:47:47 -0600936 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env,
Masahiro Yamada8f5256a2016-06-15 14:33:52 +0900937 stderr=subprocess.PIPE,
938 cwd=self.current_src_dir)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900939 self.state = STATE_AUTOCONF
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900940
Simon Glass43cf08f2017-06-01 19:39:02 -0600941 def do_build_db(self):
942 """Add the board to the database"""
943 configs = {}
Simon Glassaba238f2021-12-18 14:54:34 -0700944 for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)):
945 if line.startswith('CONFIG'):
946 config, value = line.split('=', 1)
947 configs[config] = value.rstrip()
Simon Glass43cf08f2017-06-01 19:39:02 -0600948 self.db_queue.put([self.defconfig, configs])
949 self.finish(True)
950
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900951 def do_savedefconfig(self):
952 """Update the .config and run 'make savedefconfig'."""
953
Masahiro Yamada35204d92016-08-22 22:18:21 +0900954 (updated, suspicious, log) = self.parser.update_dotconfig()
955 if suspicious:
956 self.suspicious_boards.add(self.defconfig)
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900957 self.log += log
958
Simon Glassd9c1da22021-12-18 14:54:31 -0700959 if not self.args.force_sync and not updated:
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900960 self.finish(True)
961 return
962 if updated:
Simon Glassd9c1da22021-12-18 14:54:31 -0700963 self.log += color_text(self.args.color, COLOR_LIGHT_GREEN,
Simon Glassdc634d92021-12-18 14:54:30 -0700964 'Syncing by savedefconfig...\n')
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900965 else:
Simon Glassdc634d92021-12-18 14:54:30 -0700966 self.log += 'Syncing by savedefconfig (forced by option)...\n'
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900967
968 cmd = list(self.make_cmd)
969 cmd.append('savedefconfig')
970 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
971 stderr=subprocess.PIPE)
972 self.state = STATE_SAVEDEFCONFIG
973
974 def update_defconfig(self):
975 """Update the input defconfig and go back to the idle state."""
976
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +0900977 log = self.parser.check_defconfig()
978 if log:
Masahiro Yamada1271b672016-08-22 22:18:20 +0900979 self.suspicious_boards.add(self.defconfig)
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +0900980 self.log += log
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900981 orig_defconfig = os.path.join('configs', self.defconfig)
982 new_defconfig = os.path.join(self.build_dir, 'defconfig')
983 updated = not filecmp.cmp(orig_defconfig, new_defconfig)
984
985 if updated:
Simon Glassd9c1da22021-12-18 14:54:31 -0700986 self.log += color_text(self.args.color, COLOR_LIGHT_BLUE,
Simon Glassdc634d92021-12-18 14:54:30 -0700987 'defconfig was updated.\n')
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900988
Simon Glassd9c1da22021-12-18 14:54:31 -0700989 if not self.args.dry_run and updated:
Masahiro Yamadacb256cb2016-06-08 11:47:37 +0900990 shutil.move(new_defconfig, orig_defconfig)
991 self.finish(True)
Masahiro Yamadab6160812015-05-20 11:36:07 +0900992
Masahiro Yamada274a5ee2016-05-19 15:52:03 +0900993 def finish(self, success):
994 """Display log along with progress and go to the idle state.
Masahiro Yamada465b7c02016-05-19 15:52:02 +0900995
Simon Glassb3464eb2021-12-18 14:54:35 -0700996 Args:
Masahiro Yamada274a5ee2016-05-19 15:52:03 +0900997 success: Should be True when the defconfig was processed
998 successfully, or False when it fails.
Masahiro Yamada465b7c02016-05-19 15:52:02 +0900999 """
1000 # output at least 30 characters to hide the "* defconfigs out of *".
1001 log = self.defconfig.ljust(30) + '\n'
1002
1003 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
1004 # Some threads are running in parallel.
1005 # Print log atomically to not mix up logs from different threads.
Simon Glass1f701862019-10-31 07:42:57 -06001006 print(log, file=(sys.stdout if success else sys.stderr))
Masahiro Yamada274a5ee2016-05-19 15:52:03 +09001007
1008 if not success:
Simon Glassd9c1da22021-12-18 14:54:31 -07001009 if self.args.exit_on_error:
Simon Glassdc634d92021-12-18 14:54:30 -07001010 sys.exit('Exit on error.')
Masahiro Yamada274a5ee2016-05-19 15:52:03 +09001011 # If --exit-on-error flag is not set, skip this board and continue.
1012 # Record the failed board.
Masahiro Yamada1271b672016-08-22 22:18:20 +09001013 self.failed_boards.add(self.defconfig)
Masahiro Yamada274a5ee2016-05-19 15:52:03 +09001014
Masahiro Yamada465b7c02016-05-19 15:52:02 +09001015 self.progress.inc()
1016 self.progress.show()
Masahiro Yamada274a5ee2016-05-19 15:52:03 +09001017 self.state = STATE_IDLE
Masahiro Yamada465b7c02016-05-19 15:52:02 +09001018
Masahiro Yamadab6160812015-05-20 11:36:07 +09001019 def get_failed_boards(self):
Masahiro Yamada1271b672016-08-22 22:18:20 +09001020 """Returns a set of failed boards (defconfigs) in this slot.
Masahiro Yamadab6160812015-05-20 11:36:07 +09001021 """
1022 return self.failed_boards
1023
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001024 def get_suspicious_boards(self):
Masahiro Yamada1271b672016-08-22 22:18:20 +09001025 """Returns a set of boards (defconfigs) with possible misconversion.
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001026 """
Masahiro Yamada35204d92016-08-22 22:18:21 +09001027 return self.suspicious_boards - self.failed_boards
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001028
Masahiro Yamadab6160812015-05-20 11:36:07 +09001029class Slots:
1030
1031 """Controller of the array of subprocess slots."""
1032
Simon Glassd9c1da22021-12-18 14:54:31 -07001033 def __init__(self, toolchains, configs, args, progress,
Simon Glass257f5232017-07-10 14:47:47 -06001034 reference_src_dir, db_queue):
Masahiro Yamadab6160812015-05-20 11:36:07 +09001035 """Create a new slots controller.
1036
Simon Glassb3464eb2021-12-18 14:54:35 -07001037 Args:
Simon Glass257f5232017-07-10 14:47:47 -06001038 toolchains: Toolchains object containing toolchains.
Masahiro Yamadab80a6672016-05-19 15:51:57 +09001039 configs: A list of CONFIGs to move.
Simon Glassd9c1da22021-12-18 14:54:31 -07001040 args: Program arguments
Masahiro Yamadacefaa582016-05-19 15:51:55 +09001041 progress: A progress indicator.
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001042 reference_src_dir: Determine the true starting config state from this
1043 source tree.
Simon Glass43cf08f2017-06-01 19:39:02 -06001044 db_queue: output queue to write config info for the database
Masahiro Yamadab6160812015-05-20 11:36:07 +09001045 """
Simon Glassd9c1da22021-12-18 14:54:31 -07001046 self.args = args
Masahiro Yamadab6160812015-05-20 11:36:07 +09001047 self.slots = []
Simon Glass34c505f2021-12-18 14:54:32 -07001048 devnull = subprocess.DEVNULL
Masahiro Yamadab6160812015-05-20 11:36:07 +09001049 make_cmd = get_make_cmd()
Simon Glassd9c1da22021-12-18 14:54:31 -07001050 for i in range(args.jobs):
1051 self.slots.append(Slot(toolchains, configs, args, progress,
Simon Glass257f5232017-07-10 14:47:47 -06001052 devnull, make_cmd, reference_src_dir,
1053 db_queue))
Masahiro Yamadab6160812015-05-20 11:36:07 +09001054
Masahiro Yamadacefaa582016-05-19 15:51:55 +09001055 def add(self, defconfig):
Masahiro Yamadab6160812015-05-20 11:36:07 +09001056 """Add a new subprocess if a vacant slot is found.
1057
Simon Glassb3464eb2021-12-18 14:54:35 -07001058 Args:
Masahiro Yamadab6160812015-05-20 11:36:07 +09001059 defconfig: defconfig name to be put into.
1060
1061 Returns:
1062 Return True on success or False on failure
1063 """
1064 for slot in self.slots:
Masahiro Yamadacefaa582016-05-19 15:51:55 +09001065 if slot.add(defconfig):
Masahiro Yamadab6160812015-05-20 11:36:07 +09001066 return True
1067 return False
1068
1069 def available(self):
1070 """Check if there is a vacant slot.
1071
1072 Returns:
1073 Return True if at lease one vacant slot is found, False otherwise.
1074 """
1075 for slot in self.slots:
1076 if slot.poll():
1077 return True
1078 return False
1079
1080 def empty(self):
1081 """Check if all slots are vacant.
1082
1083 Returns:
1084 Return True if all the slots are vacant, False otherwise.
1085 """
1086 ret = True
1087 for slot in self.slots:
1088 if not slot.poll():
1089 ret = False
1090 return ret
1091
1092 def show_failed_boards(self):
1093 """Display all of the failed boards (defconfigs)."""
Masahiro Yamada1271b672016-08-22 22:18:20 +09001094 boards = set()
Masahiro Yamada0153f032016-06-15 14:33:53 +09001095 output_file = 'moveconfig.failed'
Masahiro Yamadab6160812015-05-20 11:36:07 +09001096
1097 for slot in self.slots:
Masahiro Yamada1271b672016-08-22 22:18:20 +09001098 boards |= slot.get_failed_boards()
Masahiro Yamadab6160812015-05-20 11:36:07 +09001099
Masahiro Yamada0153f032016-06-15 14:33:53 +09001100 if boards:
1101 boards = '\n'.join(boards) + '\n'
Simon Glassdc634d92021-12-18 14:54:30 -07001102 msg = 'The following boards were not processed due to error:\n'
Masahiro Yamada0153f032016-06-15 14:33:53 +09001103 msg += boards
Simon Glassdc634d92021-12-18 14:54:30 -07001104 msg += '(the list has been saved in %s)\n' % output_file
Simon Glassd9c1da22021-12-18 14:54:31 -07001105 print(color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glass1f701862019-10-31 07:42:57 -06001106 msg), file=sys.stderr)
Masahiro Yamadab6160812015-05-20 11:36:07 +09001107
Simon Glassb09ae452021-12-18 14:54:33 -07001108 write_file(output_file, boards)
Joe Hershbergerdade12e2015-05-19 13:21:22 -05001109
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001110 def show_suspicious_boards(self):
1111 """Display all boards (defconfigs) with possible misconversion."""
Masahiro Yamada1271b672016-08-22 22:18:20 +09001112 boards = set()
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001113 output_file = 'moveconfig.suspicious'
1114
1115 for slot in self.slots:
Masahiro Yamada1271b672016-08-22 22:18:20 +09001116 boards |= slot.get_suspicious_boards()
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001117
1118 if boards:
1119 boards = '\n'.join(boards) + '\n'
Simon Glassdc634d92021-12-18 14:54:30 -07001120 msg = 'The following boards might have been converted incorrectly.\n'
1121 msg += 'It is highly recommended to check them manually:\n'
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001122 msg += boards
Simon Glassdc634d92021-12-18 14:54:30 -07001123 msg += '(the list has been saved in %s)\n' % output_file
Simon Glassd9c1da22021-12-18 14:54:31 -07001124 print(color_text(self.args.color, COLOR_YELLOW,
Simon Glass1f701862019-10-31 07:42:57 -06001125 msg), file=sys.stderr)
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001126
Simon Glassb09ae452021-12-18 14:54:33 -07001127 write_file(output_file, boards)
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001128
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001129class ReferenceSource:
1130
1131 """Reference source against which original configs should be parsed."""
1132
1133 def __init__(self, commit):
1134 """Create a reference source directory based on a specified commit.
1135
Simon Glassb3464eb2021-12-18 14:54:35 -07001136 Args:
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001137 commit: commit to git-clone
1138 """
1139 self.src_dir = tempfile.mkdtemp()
Simon Glassdc634d92021-12-18 14:54:30 -07001140 print('Cloning git repo to a separate work directory...')
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001141 subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
1142 cwd=self.src_dir)
Simon Glass1f701862019-10-31 07:42:57 -06001143 print("Checkout '%s' to build the original autoconf.mk." % \
1144 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip())
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001145 subprocess.check_output(['git', 'checkout', commit],
1146 stderr=subprocess.STDOUT, cwd=self.src_dir)
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001147
1148 def __del__(self):
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001149 """Delete the reference source directory
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001150
1151 This function makes sure the temporary directory is cleaned away
1152 even if Python suddenly dies due to error. It should be done in here
1153 because it is guaranteed the destructor is always invoked when the
1154 instance of the class gets unreferenced.
1155 """
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001156 shutil.rmtree(self.src_dir)
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001157
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001158 def get_dir(self):
1159 """Return the absolute path to the reference source directory."""
1160
1161 return self.src_dir
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001162
Simon Glassd9c1da22021-12-18 14:54:31 -07001163def move_config(toolchains, configs, args, db_queue):
Masahiro Yamadab6160812015-05-20 11:36:07 +09001164 """Move config options to defconfig files.
1165
Simon Glassb3464eb2021-12-18 14:54:35 -07001166 Args:
Masahiro Yamadab80a6672016-05-19 15:51:57 +09001167 configs: A list of CONFIGs to move.
Simon Glassd9c1da22021-12-18 14:54:31 -07001168 args: Program arguments
Masahiro Yamadab6160812015-05-20 11:36:07 +09001169 """
Masahiro Yamadab80a6672016-05-19 15:51:57 +09001170 if len(configs) == 0:
Simon Glassd9c1da22021-12-18 14:54:31 -07001171 if args.force_sync:
Simon Glass1f701862019-10-31 07:42:57 -06001172 print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ')
Simon Glassd9c1da22021-12-18 14:54:31 -07001173 elif args.build_db:
Simon Glass1f701862019-10-31 07:42:57 -06001174 print('Building %s database' % CONFIG_DATABASE)
Masahiro Yamada9566abd2016-05-19 15:52:09 +09001175 else:
Simon Glass1f701862019-10-31 07:42:57 -06001176 print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ')
Masahiro Yamada9566abd2016-05-19 15:52:09 +09001177 else:
Simon Glass1f701862019-10-31 07:42:57 -06001178 print('Move ' + ', '.join(configs), end=' ')
Simon Glassd9c1da22021-12-18 14:54:31 -07001179 print('(jobs: %d)\n' % args.jobs)
Masahiro Yamadab6160812015-05-20 11:36:07 +09001180
Simon Glassd9c1da22021-12-18 14:54:31 -07001181 if args.git_ref:
1182 reference_src = ReferenceSource(args.git_ref)
Masahiro Yamada2e74fee2016-06-15 14:33:51 +09001183 reference_src_dir = reference_src.get_dir()
1184 else:
Masahiro Yamada8f5256a2016-06-15 14:33:52 +09001185 reference_src_dir = None
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001186
Simon Glassd9c1da22021-12-18 14:54:31 -07001187 if args.defconfigs:
1188 defconfigs = get_matched_defconfigs(args.defconfigs)
Joe Hershbergerc6e043a2015-05-19 13:21:19 -05001189 else:
Masahiro Yamada58175e32016-07-25 19:15:28 +09001190 defconfigs = get_all_defconfigs()
Masahiro Yamadab6160812015-05-20 11:36:07 +09001191
Masahiro Yamadacefaa582016-05-19 15:51:55 +09001192 progress = Progress(len(defconfigs))
Simon Glassd9c1da22021-12-18 14:54:31 -07001193 slots = Slots(toolchains, configs, args, progress, reference_src_dir,
Simon Glassb3464eb2021-12-18 14:54:35 -07001194 db_queue)
Masahiro Yamadab6160812015-05-20 11:36:07 +09001195
1196 # Main loop to process defconfig files:
1197 # Add a new subprocess into a vacant slot.
1198 # Sleep if there is no available slot.
Masahiro Yamadacefaa582016-05-19 15:51:55 +09001199 for defconfig in defconfigs:
1200 while not slots.add(defconfig):
Masahiro Yamadab6160812015-05-20 11:36:07 +09001201 while not slots.available():
1202 # No available slot: sleep for a while
1203 time.sleep(SLEEP_TIME)
1204
1205 # wait until all the subprocesses finish
1206 while not slots.empty():
1207 time.sleep(SLEEP_TIME)
1208
Simon Glass1f701862019-10-31 07:42:57 -06001209 print('')
Masahiro Yamadab6160812015-05-20 11:36:07 +09001210 slots.show_failed_boards()
Masahiro Yamada3c9bfea2016-06-15 14:33:54 +09001211 slots.show_suspicious_boards()
Masahiro Yamadab6160812015-05-20 11:36:07 +09001212
Simon Glass44116332017-06-15 21:39:33 -06001213def find_kconfig_rules(kconf, config, imply_config):
1214 """Check whether a config has a 'select' or 'imply' keyword
1215
1216 Args:
Tom Rini3c5f4152019-09-20 17:42:09 -04001217 kconf: Kconfiglib.Kconfig object
Simon Glass44116332017-06-15 21:39:33 -06001218 config: Name of config to check (without CONFIG_ prefix)
1219 imply_config: Implying config (without CONFIG_ prefix) which may or
1220 may not have an 'imply' for 'config')
1221
1222 Returns:
1223 Symbol object for 'config' if found, else None
1224 """
Tom Rini3c5f4152019-09-20 17:42:09 -04001225 sym = kconf.syms.get(imply_config)
Simon Glass44116332017-06-15 21:39:33 -06001226 if sym:
Simon Glass520b47a2021-07-21 21:35:53 -06001227 for sel, cond in (sym.selects + sym.implies):
Simon Glass93c0a9e2021-12-18 08:09:42 -07001228 if sel.name == config:
Simon Glass44116332017-06-15 21:39:33 -06001229 return sym
1230 return None
1231
1232def check_imply_rule(kconf, config, imply_config):
1233 """Check if we can add an 'imply' option
1234
1235 This finds imply_config in the Kconfig and looks to see if it is possible
1236 to add an 'imply' for 'config' to that part of the Kconfig.
1237
1238 Args:
Tom Rini3c5f4152019-09-20 17:42:09 -04001239 kconf: Kconfiglib.Kconfig object
Simon Glass44116332017-06-15 21:39:33 -06001240 config: Name of config to check (without CONFIG_ prefix)
1241 imply_config: Implying config (without CONFIG_ prefix) which may or
1242 may not have an 'imply' for 'config')
1243
1244 Returns:
1245 tuple:
1246 filename of Kconfig file containing imply_config, or None if none
1247 line number within the Kconfig file, or 0 if none
1248 message indicating the result
1249 """
Tom Rini3c5f4152019-09-20 17:42:09 -04001250 sym = kconf.syms.get(imply_config)
Simon Glass44116332017-06-15 21:39:33 -06001251 if not sym:
1252 return 'cannot find sym'
Simon Glass520b47a2021-07-21 21:35:53 -06001253 nodes = sym.nodes
1254 if len(nodes) != 1:
1255 return '%d locations' % len(nodes)
Simon Glass93c0a9e2021-12-18 08:09:42 -07001256 node = nodes[0]
1257 fname, linenum = node.filename, node.linenr
Simon Glass44116332017-06-15 21:39:33 -06001258 cwd = os.getcwd()
1259 if cwd and fname.startswith(cwd):
1260 fname = fname[len(cwd) + 1:]
1261 file_line = ' at %s:%d' % (fname, linenum)
Simon Glassaba238f2021-12-18 14:54:34 -07001262 data = read_file(fname)
Simon Glass44116332017-06-15 21:39:33 -06001263 if data[linenum - 1] != 'config %s' % imply_config:
1264 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line)
1265 return fname, linenum, 'adding%s' % file_line
1266
1267def add_imply_rule(config, fname, linenum):
1268 """Add a new 'imply' option to a Kconfig
1269
1270 Args:
1271 config: config option to add an imply for (without CONFIG_ prefix)
1272 fname: Kconfig filename to update
1273 linenum: Line number to place the 'imply' before
1274
1275 Returns:
1276 Message indicating the result
1277 """
1278 file_line = ' at %s:%d' % (fname, linenum)
Simon Glassaba238f2021-12-18 14:54:34 -07001279 data = read_file(fname)
Simon Glass44116332017-06-15 21:39:33 -06001280 linenum -= 1
1281
1282 for offset, line in enumerate(data[linenum:]):
1283 if line.strip().startswith('help') or not line:
1284 data.insert(linenum + offset, '\timply %s' % config)
Simon Glassb09ae452021-12-18 14:54:33 -07001285 write_file(fname, data)
Simon Glass44116332017-06-15 21:39:33 -06001286 return 'added%s' % file_line
1287
1288 return 'could not insert%s'
1289
1290(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
1291 1, 2, 4, 8)
Simon Glass92e55582017-06-15 21:39:32 -06001292
1293IMPLY_FLAGS = {
1294 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
1295 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
1296 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
Simon Glass44116332017-06-15 21:39:33 -06001297 'non-arch-board': [
1298 IMPLY_NON_ARCH_BOARD,
1299 'Allow Kconfig options outside arch/ and /board/ to imply'],
Simon Glassb3464eb2021-12-18 14:54:35 -07001300}
Simon Glass92e55582017-06-15 21:39:32 -06001301
Simon Glassf931c2f2021-12-18 08:09:43 -07001302
1303def read_database():
1304 """Read in the config database
1305
1306 Returns:
1307 tuple:
1308 set of all config options seen (each a str)
1309 set of all defconfigs seen (each a str)
1310 dict of configs for each defconfig:
1311 key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig"
1312 value: dict:
1313 key: CONFIG option
1314 value: Value of option
1315 dict of defconfigs for each config:
1316 key: CONFIG option
1317 value: set of boards using that option
1318
1319 """
1320 configs = {}
1321
1322 # key is defconfig name, value is dict of (CONFIG_xxx, value)
1323 config_db = {}
1324
1325 # Set of all config options we have seen
1326 all_configs = set()
1327
1328 # Set of all defconfigs we have seen
1329 all_defconfigs = set()
1330
1331 defconfig_db = collections.defaultdict(set)
Simon Glassaba238f2021-12-18 14:54:34 -07001332 for line in read_file(CONFIG_DATABASE):
1333 line = line.rstrip()
1334 if not line: # Separator between defconfigs
1335 config_db[defconfig] = configs
1336 all_defconfigs.add(defconfig)
1337 configs = {}
1338 elif line[0] == ' ': # CONFIG line
1339 config, value = line.strip().split('=', 1)
1340 configs[config] = value
1341 defconfig_db[config].add(defconfig)
1342 all_configs.add(config)
1343 else: # New defconfig
1344 defconfig = line
Simon Glassf931c2f2021-12-18 08:09:43 -07001345
1346 return all_configs, all_defconfigs, config_db, defconfig_db
1347
1348
Simon Glass44116332017-06-15 21:39:33 -06001349def do_imply_config(config_list, add_imply, imply_flags, skip_added,
1350 check_kconfig=True, find_superset=False):
Simon Glassc6e73cf2017-06-01 19:39:03 -06001351 """Find CONFIG options which imply those in the list
1352
1353 Some CONFIG options can be implied by others and this can help to reduce
1354 the size of the defconfig files. For example, CONFIG_X86 implies
1355 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
1356 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
1357 each of the x86 defconfig files.
1358
1359 This function uses the moveconfig database to find such options. It
1360 displays a list of things that could possibly imply those in the list.
1361 The algorithm ignores any that start with CONFIG_TARGET since these
1362 typically refer to only a few defconfigs (often one). It also does not
1363 display a config with less than 5 defconfigs.
1364
1365 The algorithm works using sets. For each target config in config_list:
1366 - Get the set 'defconfigs' which use that target config
1367 - For each config (from a list of all configs):
1368 - Get the set 'imply_defconfig' of defconfigs which use that config
1369 -
1370 - If imply_defconfigs contains anything not in defconfigs then
1371 this config does not imply the target config
1372
1373 Params:
1374 config_list: List of CONFIG options to check (each a string)
Simon Glass44116332017-06-15 21:39:33 -06001375 add_imply: Automatically add an 'imply' for each config.
Simon Glass92e55582017-06-15 21:39:32 -06001376 imply_flags: Flags which control which implying configs are allowed
1377 (IMPLY_...)
Simon Glass44116332017-06-15 21:39:33 -06001378 skip_added: Don't show options which already have an imply added.
1379 check_kconfig: Check if implied symbols already have an 'imply' or
1380 'select' for the target config, and show this information if so.
Simon Glassc6e73cf2017-06-01 19:39:03 -06001381 find_superset: True to look for configs which are a superset of those
1382 already found. So for example if CONFIG_EXYNOS5 implies an option,
1383 but CONFIG_EXYNOS covers a larger set of defconfigs and also
1384 implies that option, this will drop the former in favour of the
1385 latter. In practice this option has not proved very used.
1386
1387 Note the terminoloy:
1388 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
1389 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
1390 """
Simon Glass44116332017-06-15 21:39:33 -06001391 kconf = KconfigScanner().conf if check_kconfig else None
1392 if add_imply and add_imply != 'all':
Simon Glass93c0a9e2021-12-18 08:09:42 -07001393 add_imply = add_imply.split(',')
Simon Glass44116332017-06-15 21:39:33 -06001394
Simon Glassf931c2f2021-12-18 08:09:43 -07001395 all_configs, all_defconfigs, config_db, defconfig_db = read_database()
Simon Glassc6e73cf2017-06-01 19:39:03 -06001396
Simon Glass93c0a9e2021-12-18 08:09:42 -07001397 # Work through each target config option in turn, independently
Simon Glassc6e73cf2017-06-01 19:39:03 -06001398 for config in config_list:
1399 defconfigs = defconfig_db.get(config)
1400 if not defconfigs:
Simon Glass1f701862019-10-31 07:42:57 -06001401 print('%s not found in any defconfig' % config)
Simon Glassc6e73cf2017-06-01 19:39:03 -06001402 continue
1403
1404 # Get the set of defconfigs without this one (since a config cannot
1405 # imply itself)
1406 non_defconfigs = all_defconfigs - defconfigs
1407 num_defconfigs = len(defconfigs)
Simon Glass1f701862019-10-31 07:42:57 -06001408 print('%s found in %d/%d defconfigs' % (config, num_defconfigs,
1409 len(all_configs)))
Simon Glassc6e73cf2017-06-01 19:39:03 -06001410
1411 # This will hold the results: key=config, value=defconfigs containing it
1412 imply_configs = {}
1413 rest_configs = all_configs - set([config])
1414
1415 # Look at every possible config, except the target one
1416 for imply_config in rest_configs:
Simon Glass92e55582017-06-15 21:39:32 -06001417 if 'ERRATUM' in imply_config:
Simon Glassc6e73cf2017-06-01 19:39:03 -06001418 continue
Simon Glassb3464eb2021-12-18 14:54:35 -07001419 if not imply_flags & IMPLY_CMD:
Simon Glass92e55582017-06-15 21:39:32 -06001420 if 'CONFIG_CMD' in imply_config:
1421 continue
Simon Glassb3464eb2021-12-18 14:54:35 -07001422 if not imply_flags & IMPLY_TARGET:
Simon Glass92e55582017-06-15 21:39:32 -06001423 if 'CONFIG_TARGET' in imply_config:
1424 continue
Simon Glassc6e73cf2017-06-01 19:39:03 -06001425
1426 # Find set of defconfigs that have this config
1427 imply_defconfig = defconfig_db[imply_config]
1428
1429 # Get the intersection of this with defconfigs containing the
1430 # target config
1431 common_defconfigs = imply_defconfig & defconfigs
1432
1433 # Get the set of defconfigs containing this config which DO NOT
1434 # also contain the taret config. If this set is non-empty it means
1435 # that this config affects other defconfigs as well as (possibly)
1436 # the ones affected by the target config. This means it implies
1437 # things we don't want to imply.
1438 not_common_defconfigs = imply_defconfig & non_defconfigs
1439 if not_common_defconfigs:
1440 continue
1441
1442 # If there are common defconfigs, imply_config may be useful
1443 if common_defconfigs:
1444 skip = False
1445 if find_superset:
Simon Glass1f701862019-10-31 07:42:57 -06001446 for prev in list(imply_configs.keys()):
Simon Glassc6e73cf2017-06-01 19:39:03 -06001447 prev_count = len(imply_configs[prev])
1448 count = len(common_defconfigs)
1449 if (prev_count > count and
1450 (imply_configs[prev] & common_defconfigs ==
1451 common_defconfigs)):
1452 # skip imply_config because prev is a superset
1453 skip = True
1454 break
1455 elif count > prev_count:
1456 # delete prev because imply_config is a superset
1457 del imply_configs[prev]
1458 if not skip:
1459 imply_configs[imply_config] = common_defconfigs
1460
1461 # Now we have a dict imply_configs of configs which imply each config
1462 # The value of each dict item is the set of defconfigs containing that
1463 # config. Rank them so that we print the configs that imply the largest
1464 # number of defconfigs first.
Simon Glass44116332017-06-15 21:39:33 -06001465 ranked_iconfigs = sorted(imply_configs,
Simon Glassc6e73cf2017-06-01 19:39:03 -06001466 key=lambda k: len(imply_configs[k]), reverse=True)
Simon Glass44116332017-06-15 21:39:33 -06001467 kconfig_info = ''
1468 cwd = os.getcwd()
1469 add_list = collections.defaultdict(list)
1470 for iconfig in ranked_iconfigs:
1471 num_common = len(imply_configs[iconfig])
Simon Glassc6e73cf2017-06-01 19:39:03 -06001472
1473 # Don't bother if there are less than 5 defconfigs affected.
Simon Glass92e55582017-06-15 21:39:32 -06001474 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
Simon Glassc6e73cf2017-06-01 19:39:03 -06001475 continue
Simon Glass44116332017-06-15 21:39:33 -06001476 missing = defconfigs - imply_configs[iconfig]
Simon Glassc6e73cf2017-06-01 19:39:03 -06001477 missing_str = ', '.join(missing) if missing else 'all'
1478 missing_str = ''
Simon Glass44116332017-06-15 21:39:33 -06001479 show = True
1480 if kconf:
1481 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
1482 iconfig[CONFIG_LEN:])
1483 kconfig_info = ''
1484 if sym:
Simon Glass520b47a2021-07-21 21:35:53 -06001485 nodes = sym.nodes
1486 if len(nodes) == 1:
1487 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glass44116332017-06-15 21:39:33 -06001488 if cwd and fname.startswith(cwd):
1489 fname = fname[len(cwd) + 1:]
1490 kconfig_info = '%s:%d' % (fname, linenum)
1491 if skip_added:
1492 show = False
1493 else:
Tom Rini3c5f4152019-09-20 17:42:09 -04001494 sym = kconf.syms.get(iconfig[CONFIG_LEN:])
Simon Glass44116332017-06-15 21:39:33 -06001495 fname = ''
1496 if sym:
Simon Glass520b47a2021-07-21 21:35:53 -06001497 nodes = sym.nodes
1498 if len(nodes) == 1:
1499 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glass44116332017-06-15 21:39:33 -06001500 if cwd and fname.startswith(cwd):
1501 fname = fname[len(cwd) + 1:]
1502 in_arch_board = not sym or (fname.startswith('arch') or
1503 fname.startswith('board'))
1504 if (not in_arch_board and
Simon Glassb3464eb2021-12-18 14:54:35 -07001505 not imply_flags & IMPLY_NON_ARCH_BOARD):
Simon Glass44116332017-06-15 21:39:33 -06001506 continue
1507
1508 if add_imply and (add_imply == 'all' or
1509 iconfig in add_imply):
1510 fname, linenum, kconfig_info = (check_imply_rule(kconf,
1511 config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
1512 if fname:
1513 add_list[fname].append(linenum)
Simon Glassc6e73cf2017-06-01 19:39:03 -06001514
Simon Glass44116332017-06-15 21:39:33 -06001515 if show and kconfig_info != 'skip':
Simon Glass1f701862019-10-31 07:42:57 -06001516 print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
1517 kconfig_info, missing_str))
Simon Glass44116332017-06-15 21:39:33 -06001518
1519 # Having collected a list of things to add, now we add them. We process
1520 # each file from the largest line number to the smallest so that
1521 # earlier additions do not affect our line numbers. E.g. if we added an
1522 # imply at line 20 it would change the position of each line after
1523 # that.
Simon Glass1f701862019-10-31 07:42:57 -06001524 for fname, linenums in add_list.items():
Simon Glass44116332017-06-15 21:39:33 -06001525 for linenum in sorted(linenums, reverse=True):
1526 add_imply_rule(config[CONFIG_LEN:], fname, linenum)
1527
Simon Glass99f79422022-02-08 11:49:46 -07001528def defconfig_matches(configs, re_match):
1529 """Check if any CONFIG option matches a regex
1530
1531 The match must be complete, i.e. from the start to end of the CONFIG option.
1532
1533 Args:
1534 configs (dict): Dict of CONFIG options:
1535 key: CONFIG option
1536 value: Value of option
1537 re_match (re.Pattern): Match to check
1538
1539 Returns:
1540 bool: True if any CONFIG matches the regex
1541 """
1542 for cfg in configs:
Simon Glassfea71c92022-03-05 20:18:54 -07001543 if re_match.fullmatch(cfg):
Simon Glass99f79422022-02-08 11:49:46 -07001544 return True
1545 return False
Simon Glassc6e73cf2017-06-01 19:39:03 -06001546
Simon Glass0082b2e2021-12-18 08:09:46 -07001547def do_find_config(config_list):
1548 """Find boards with a given combination of CONFIGs
1549
1550 Params:
Simon Glass99f79422022-02-08 11:49:46 -07001551 config_list: List of CONFIG options to check (each a regex consisting
Simon Glass0082b2e2021-12-18 08:09:46 -07001552 of a config option, with or without a CONFIG_ prefix. If an option
1553 is preceded by a tilde (~) then it must be false, otherwise it must
1554 be true)
1555 """
1556 all_configs, all_defconfigs, config_db, defconfig_db = read_database()
1557
Simon Glass0082b2e2021-12-18 08:09:46 -07001558 # Start with all defconfigs
1559 out = all_defconfigs
1560
1561 # Work through each config in turn
Simon Glass0082b2e2021-12-18 08:09:46 -07001562 for item in config_list:
1563 # Get the real config name and whether we want this config or not
1564 cfg = item
1565 want = True
1566 if cfg[0] == '~':
1567 want = False
1568 cfg = cfg[1:]
1569
Simon Glass0082b2e2021-12-18 08:09:46 -07001570 # Search everything that is still in the running. If it has a config
1571 # that we want, or doesn't have one that we don't, add it into the
1572 # running for the next stage
1573 in_list = out
1574 out = set()
Simon Glass99f79422022-02-08 11:49:46 -07001575 re_match = re.compile(cfg)
Simon Glass0082b2e2021-12-18 08:09:46 -07001576 for defc in in_list:
Simon Glass99f79422022-02-08 11:49:46 -07001577 has_cfg = defconfig_matches(config_db[defc], re_match)
Simon Glass0082b2e2021-12-18 08:09:46 -07001578 if has_cfg == want:
1579 out.add(defc)
Tom Rinic1b64aa2022-12-04 10:14:16 -05001580 print(f'{len(out)} matches')
1581 print(' '.join(item.split('_defconfig')[0] for item in out))
Simon Glass0082b2e2021-12-18 08:09:46 -07001582
1583
1584def prefix_config(cfg):
1585 """Prefix a config with CONFIG_ if needed
1586
1587 This handles ~ operator, which indicates that the CONFIG should be disabled
1588
1589 >>> prefix_config('FRED')
1590 'CONFIG_FRED'
1591 >>> prefix_config('CONFIG_FRED')
1592 'CONFIG_FRED'
1593 >>> prefix_config('~FRED')
1594 '~CONFIG_FRED'
1595 >>> prefix_config('~CONFIG_FRED')
1596 '~CONFIG_FRED'
1597 >>> prefix_config('A123')
1598 'CONFIG_A123'
1599 """
1600 op = ''
1601 if cfg[0] == '~':
1602 op = cfg[0]
1603 cfg = cfg[1:]
1604 if not cfg.startswith('CONFIG_'):
1605 cfg = 'CONFIG_' + cfg
1606 return op + cfg
1607
Simon Glass4c4eb7c2023-02-01 13:19:12 -07001608
1609RE_MK_CONFIGS = re.compile('CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
1610RE_IFDEF = re.compile('(ifdef|ifndef)')
1611RE_C_CONFIGS = re.compile('CONFIG_([A-Za-z0-9_]*)')
1612RE_CONFIG_IS = re.compile('CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
1613
1614class ConfigUse:
1615 def __init__(self, cfg, is_spl, fname, rest):
1616 self.cfg = cfg
1617 self.is_spl = is_spl
1618 self.fname = fname
1619 self.rest = rest
1620
1621 def __hash__(self):
1622 return hash((self.cfg, self.is_spl))
1623
1624def scan_makefiles(fnames):
1625 """Scan Makefiles looking for Kconfig options
1626
1627 Looks for uses of CONFIG options in Makefiles
1628
1629 Args:
1630 fnames (list of tuple):
1631 str: Makefile filename where the option was found
1632 str: Line of the Makefile
1633
1634 Returns:
1635 tuple:
1636 dict: all_uses
1637 key (ConfigUse): object
1638 value (list of str): matching lines
1639 dict: Uses by filename
1640 key (str): filename
1641 value (set of ConfigUse): uses in that filename
1642
1643 >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups()
1644 (None, 'FRED')
1645 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups()
1646 ('$(SPL_)', 'MARY')
1647 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
1648 ('$(SPL_TPL_)', 'MARY')
1649 """
1650 all_uses = collections.defaultdict(list)
1651 fname_uses = {}
1652 for fname, rest in fnames:
1653 m_iter = RE_MK_CONFIGS.finditer(rest)
1654 found = False
1655 for m in m_iter:
1656 found = True
1657 real_opt = m.group(2)
1658 if real_opt == '':
1659 continue
1660 is_spl = False
1661 if m.group(1):
1662 is_spl = True
1663 use = ConfigUse(real_opt, is_spl, fname, rest)
1664 if fname not in fname_uses:
1665 fname_uses[fname] = set()
1666 fname_uses[fname].add(use)
1667 all_uses[use].append(rest)
1668 return all_uses, fname_uses
1669
1670
1671def scan_src_files(fnames):
1672 """Scan source files (other than Makefiles) looking for Kconfig options
1673
1674 Looks for uses of CONFIG options
1675
1676 Args:
1677 fnames (list of tuple):
1678 str: Makefile filename where the option was found
1679 str: Line of the Makefile
1680
1681 Returns:
1682 tuple:
1683 dict: all_uses
1684 key (ConfigUse): object
1685 value (list of str): matching lines
1686 dict: Uses by filename
1687 key (str): filename
1688 value (set of ConfigUse): uses in that filename
1689
1690 >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
1691 ('FRED',)
1692 >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups()
1693 ('MARY',)
1694 >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
1695 ('OF_PLATDATA',)
1696 """
1697 def add_uses(m_iter, is_spl):
1698 for m in m_iter:
1699 found = True
1700 real_opt = m.group(1)
1701 if real_opt == '':
1702 continue
1703 use = ConfigUse(real_opt, is_spl, fname, rest)
1704 if fname not in fname_uses:
1705 fname_uses[fname] = set()
1706 fname_uses[fname].add(use)
1707 all_uses[use].append(rest)
1708
1709 all_uses = collections.defaultdict(list)
1710 fname_uses = {}
1711 for fname, rest in fnames:
1712 m_iter = RE_C_CONFIGS.finditer(rest)
1713 add_uses(m_iter, False)
1714
1715 m_iter2 = RE_CONFIG_IS.finditer(rest)
1716 add_uses(m_iter2, True)
1717
1718 return all_uses, fname_uses
1719
1720
1721MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
1722
1723def do_scan_source(path, do_update):
1724 """Scan the source tree for Kconfig inconsistencies
1725
1726 Args:
1727 path (str): Path to source tree
1728 do_update (bool) : True to write to scripts/kconf_... files
1729 """
1730 def is_not_proper(name):
1731 for prefix in SPL_PREFIXES:
1732 if name.startswith(prefix):
1733 return name[len(prefix):]
1734 return False
1735
1736 def check_not_found(all_uses, spl_mode):
1737 """Check for Kconfig options mentioned in the source but not in Kconfig
1738
1739 Args:
1740 all_uses (dict):
1741 key (ConfigUse): object
1742 value (list of str): matching lines
1743 spl_mode (int): If MODE_SPL, look at source code which implies
1744 an SPL_ option, but for which there is none;
1745 for MOD_PROPER, look at source code which implies a Proper
1746 option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or
1747 $(SPL_TPL_) but for which there none;
1748 if MODE_NORMAL, ignore SPL
1749
1750 Returns:
1751 dict:
1752 key (str): CONFIG name (without 'CONFIG_' prefix
1753 value (list of ConfigUse): List of uses of this CONFIG
1754 """
1755 # Make sure we know about all the options
1756 not_found = collections.defaultdict(list)
1757 for use, rest in all_uses.items():
1758 name = use.cfg
1759 if name in IGNORE_SYMS:
1760 continue
1761 check = True
1762
1763 if spl_mode == MODE_SPL:
1764 check = use.is_spl
1765
1766 # If it is an SPL symbol, try prepending all SPL_ prefixes to
1767 # find at least one SPL symbol
1768 if use.is_spl:
1769 add_to_dict = False
1770 for prefix in SPL_PREFIXES:
1771 try_name = prefix + name
1772 sym = kconf.syms.get(try_name)
1773 if sym:
1774 break
1775 if not sym:
1776 not_found[f'SPL_{name}'].append(use)
1777 continue
1778 elif spl_mode == MODE_PROPER:
1779 # Try to find the Proper version of this symbol, i.e. without
1780 # the SPL_ prefix
1781 proper_name = is_not_proper(name)
1782 if proper_name:
1783 name = proper_name
1784 elif not use.is_spl:
1785 check = False
1786 else: # MODE_NORMAL
1787 debug = False
1788 sym = kconf.syms.get(name)
1789 if not sym:
1790 proper_name = is_not_proper(name)
1791 if proper_name:
1792 name = proper_name
1793 sym = kconf.syms.get(name)
1794 if not sym:
1795 for prefix in SPL_PREFIXES:
1796 try_name = prefix + name
1797 sym = kconf.syms.get(try_name)
1798 if sym:
1799 break
1800 if not sym:
1801 not_found[name].append(use)
1802 continue
1803
1804 sym = kconf.syms.get(name)
1805 if not sym and check:
1806 not_found[name].append(use)
1807 return not_found
1808
1809 def show_uses(uses):
1810 """Show a list of uses along with their filename and code snippet
1811
1812 Args:
1813 uses (dict):
1814 key (str): CONFIG name (without 'CONFIG_' prefix
1815 value (list of ConfigUse): List of uses of this CONFIG
1816 """
1817 for name in sorted(uses):
1818 print(f'{name}: ', end='')
1819 for i, use in enumerate(uses[name]):
1820 print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}')
1821
1822
1823 print('Scanning Kconfig')
1824 kconf = KconfigScanner().conf
1825 print(f'Scanning source in {path}')
1826 args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
1827 with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
1828 out, err = proc.communicate()
1829 lines = out.splitlines()
1830 re_fname = re.compile('^([^:]*):(.*)')
1831 src_list = []
1832 mk_list = []
1833 for line in lines:
1834 linestr = line.decode('utf-8')
1835 m_fname = re_fname.search(linestr)
1836 if not m_fname:
1837 continue
1838 fname, rest = m_fname.groups()
1839 dirname, leaf = os.path.split(fname)
1840 root, ext = os.path.splitext(leaf)
1841 if ext == '.autoconf':
1842 pass
1843 elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
1844 '.env', '.tmpl']:
1845 src_list.append([fname, rest])
1846 elif 'Makefile' in root or ext == '.mk':
1847 mk_list.append([fname, rest])
1848 elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
1849 pass
1850 elif 'Kconfig' in root or 'Kbuild' in root:
1851 pass
1852 elif 'README' in root:
1853 pass
1854 elif dirname in ['configs']:
1855 pass
1856 elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
1857 pass
1858 else:
1859 print(f'Not sure how to handle file {fname}')
1860
1861 # Scan the Makefiles
1862 all_uses, fname_uses = scan_makefiles(mk_list)
1863
1864 spl_not_found = set()
1865 proper_not_found = set()
1866
1867 # Make sure we know about all the options
1868 print('\nCONFIG options present in Makefiles but not Kconfig:')
1869 not_found = check_not_found(all_uses, MODE_NORMAL)
1870 show_uses(not_found)
1871
1872 print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
1873 not_found = check_not_found(all_uses, MODE_SPL)
1874 show_uses(not_found)
1875 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1876
1877 print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
1878 not_found = check_not_found(all_uses, MODE_PROPER)
1879 show_uses(not_found)
1880 proper_not_found |= set([key for key in not_found.keys()])
1881
1882 # Scan the source code
1883 all_uses, fname_uses = scan_src_files(src_list)
1884
1885 # Make sure we know about all the options
1886 print('\nCONFIG options present in source but not Kconfig:')
1887 not_found = check_not_found(all_uses, MODE_NORMAL)
1888 show_uses(not_found)
1889
1890 print('\nCONFIG options present in source but not Kconfig (SPL):')
1891 not_found = check_not_found(all_uses, MODE_SPL)
1892 show_uses(not_found)
1893 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1894
1895 print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
1896 not_found = check_not_found(all_uses, MODE_PROPER)
1897 show_uses(not_found)
1898 proper_not_found |= set([key for key in not_found.keys()])
1899
1900 print('\nCONFIG options used as SPL but without an SPL_ variant:')
1901 for item in sorted(spl_not_found):
1902 print(f' {item}')
1903
1904 print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
1905 for item in sorted(proper_not_found):
1906 print(f' {item}')
1907
1908 # Write out the updated information
1909 if do_update:
1910 with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w') as out:
1911 print('# These options should not be enabled in SPL builds\n',
1912 file=out)
1913 for item in sorted(spl_not_found):
1914 print(item, file=out)
1915 with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w') as out:
1916 print('# These options should not be enabled in Proper builds\n',
1917 file=out)
1918 for item in sorted(proper_not_found):
1919 print(item, file=out)
1920
Simon Glass0082b2e2021-12-18 08:09:46 -07001921
Masahiro Yamadab6160812015-05-20 11:36:07 +09001922def main():
1923 try:
1924 cpu_count = multiprocessing.cpu_count()
1925 except NotImplementedError:
1926 cpu_count = 1
1927
Simon Glassd9c1da22021-12-18 14:54:31 -07001928 epilog = '''Move config options from headers to defconfig files. See
1929doc/develop/moveconfig.rst for documentation.'''
1930
1931 parser = ArgumentParser(epilog=epilog)
1932 # Add arguments here
1933 parser.add_argument('-a', '--add-imply', type=str, default='',
Simon Glass44116332017-06-15 21:39:33 -06001934 help='comma-separated list of CONFIG options to add '
1935 "an 'imply' statement to for the CONFIG in -i")
Simon Glassd9c1da22021-12-18 14:54:31 -07001936 parser.add_argument('-A', '--skip-added', action='store_true', default=False,
Simon Glass44116332017-06-15 21:39:33 -06001937 help="don't show options which are already marked as "
1938 'implying others')
Simon Glassd9c1da22021-12-18 14:54:31 -07001939 parser.add_argument('-b', '--build-db', action='store_true', default=False,
Simon Glass43cf08f2017-06-01 19:39:02 -06001940 help='build a CONFIG database')
Simon Glassd9c1da22021-12-18 14:54:31 -07001941 parser.add_argument('-c', '--color', action='store_true', default=False,
Masahiro Yamadab6160812015-05-20 11:36:07 +09001942 help='display the log in color')
Simon Glassd9c1da22021-12-18 14:54:31 -07001943 parser.add_argument('-C', '--commit', action='store_true', default=False,
Simon Glass8bf41c22016-09-12 23:18:21 -06001944 help='Create a git commit for the operation')
Simon Glassd9c1da22021-12-18 14:54:31 -07001945 parser.add_argument('-d', '--defconfigs', type=str,
Simon Glass8f3cf312017-06-01 19:38:59 -06001946 help='a file containing a list of defconfigs to move, '
1947 "one per line (for example 'snow_defconfig') "
1948 "or '-' to read from stdin")
Simon Glassd9c1da22021-12-18 14:54:31 -07001949 parser.add_argument('-e', '--exit-on-error', action='store_true',
Masahiro Yamadab6160812015-05-20 11:36:07 +09001950 default=False,
1951 help='exit immediately on any error')
Simon Glassd9c1da22021-12-18 14:54:31 -07001952 parser.add_argument('-f', '--find', action='store_true', default=False,
Simon Glass0082b2e2021-12-18 08:09:46 -07001953 help='Find boards with a given config combination')
Simon Glassd9c1da22021-12-18 14:54:31 -07001954 parser.add_argument('-H', '--headers-only', dest='cleanup_headers_only',
Joe Hershberger23475932015-05-19 13:21:20 -05001955 action='store_true', default=False,
1956 help='only cleanup the headers')
Simon Glassd9c1da22021-12-18 14:54:31 -07001957 parser.add_argument('-i', '--imply', action='store_true', default=False,
Simon Glass0559a742021-12-18 08:09:44 -07001958 help='find options which imply others')
Simon Glassd9c1da22021-12-18 14:54:31 -07001959 parser.add_argument('-I', '--imply-flags', type=str, default='',
Simon Glass0559a742021-12-18 08:09:44 -07001960 help="control the -i option ('help' for help")
Simon Glassd9c1da22021-12-18 14:54:31 -07001961 parser.add_argument('-j', '--jobs', type=int, default=cpu_count,
Masahiro Yamadab6160812015-05-20 11:36:07 +09001962 help='the number of jobs to run simultaneously')
Simon Glassd9c1da22021-12-18 14:54:31 -07001963 parser.add_argument('-n', '--dry-run', action='store_true', default=False,
Simon Glass0559a742021-12-18 08:09:44 -07001964 help='perform a trial run (show log with no changes)')
Simon Glassd9c1da22021-12-18 14:54:31 -07001965 parser.add_argument('-r', '--git-ref', type=str,
Joe Hershbergerb1a570f2016-06-10 14:53:32 -05001966 help='the git ref to clone for building the autoconf.mk')
Simon Glassd9c1da22021-12-18 14:54:31 -07001967 parser.add_argument('-s', '--force-sync', action='store_true', default=False,
Simon Glass0559a742021-12-18 08:09:44 -07001968 help='force sync by savedefconfig')
Simon Glassd9c1da22021-12-18 14:54:31 -07001969 parser.add_argument('-S', '--spl', action='store_true', default=False,
Simon Glass0559a742021-12-18 08:09:44 -07001970 help='parse config options defined for SPL build')
Simon Glass4c4eb7c2023-02-01 13:19:12 -07001971 parser.add_argument('--scan-source', action='store_true', default=False,
1972 help='scan source for uses of CONFIG options')
Simon Glassd9c1da22021-12-18 14:54:31 -07001973 parser.add_argument('-t', '--test', action='store_true', default=False,
Simon Glass0559a742021-12-18 08:09:44 -07001974 help='run unit tests')
Simon Glassd9c1da22021-12-18 14:54:31 -07001975 parser.add_argument('-y', '--yes', action='store_true', default=False,
Simon Glass13e05a02016-09-12 23:18:20 -06001976 help="respond 'yes' to any prompts")
Simon Glass4c4eb7c2023-02-01 13:19:12 -07001977 parser.add_argument('-u', '--update', action='store_true', default=False,
1978 help="update scripts/ files (use with --scan-source)")
Simon Glassd9c1da22021-12-18 14:54:31 -07001979 parser.add_argument('-v', '--verbose', action='store_true', default=False,
Joe Hershberger808b63f2015-05-19 13:21:24 -05001980 help='show any build errors as boards are built')
Simon Glassd9c1da22021-12-18 14:54:31 -07001981 parser.add_argument('configs', nargs='*')
Masahiro Yamadab6160812015-05-20 11:36:07 +09001982
Simon Glassd9c1da22021-12-18 14:54:31 -07001983 args = parser.parse_args()
1984 configs = args.configs
Masahiro Yamadab6160812015-05-20 11:36:07 +09001985
Simon Glassd9c1da22021-12-18 14:54:31 -07001986 if args.test:
Simon Glassbb57be72021-12-18 08:09:45 -07001987 sys.argv = [sys.argv[0]]
1988 fail, count = doctest.testmod()
1989 if fail:
1990 return 1
1991 unittest.main()
1992
Simon Glass4c4eb7c2023-02-01 13:19:12 -07001993 if args.scan_source:
1994 do_scan_source(os.getcwd(), args.update)
1995 return
1996
Simon Glassd9c1da22021-12-18 14:54:31 -07001997 if not any((len(configs), args.force_sync, args.build_db, args.imply,
1998 args.find)):
Masahiro Yamadab6160812015-05-20 11:36:07 +09001999 parser.print_usage()
2000 sys.exit(1)
2001
Masahiro Yamadab903c4e2016-05-19 15:51:58 +09002002 # prefix the option name with CONFIG_ if missing
Simon Glass0082b2e2021-12-18 08:09:46 -07002003 configs = [prefix_config(cfg) for cfg in configs]
Masahiro Yamadab6160812015-05-20 11:36:07 +09002004
Joe Hershberger23475932015-05-19 13:21:20 -05002005 check_top_directory()
2006
Simon Glassd9c1da22021-12-18 14:54:31 -07002007 if args.imply:
Simon Glass92e55582017-06-15 21:39:32 -06002008 imply_flags = 0
Simon Glassd9c1da22021-12-18 14:54:31 -07002009 if args.imply_flags == 'all':
Simon Glass5f096922017-07-10 14:47:46 -06002010 imply_flags = -1
2011
Simon Glassd9c1da22021-12-18 14:54:31 -07002012 elif args.imply_flags:
2013 for flag in args.imply_flags.split(','):
Simon Glass5f096922017-07-10 14:47:46 -06002014 bad = flag not in IMPLY_FLAGS
2015 if bad:
Simon Glass1f701862019-10-31 07:42:57 -06002016 print("Invalid flag '%s'" % flag)
Simon Glass5f096922017-07-10 14:47:46 -06002017 if flag == 'help' or bad:
Simon Glass1f701862019-10-31 07:42:57 -06002018 print("Imply flags: (separate with ',')")
2019 for name, info in IMPLY_FLAGS.items():
2020 print(' %-15s: %s' % (name, info[1]))
Simon Glass5f096922017-07-10 14:47:46 -06002021 parser.print_usage()
2022 sys.exit(1)
2023 imply_flags |= IMPLY_FLAGS[flag][0]
Simon Glass92e55582017-06-15 21:39:32 -06002024
Simon Glassd9c1da22021-12-18 14:54:31 -07002025 do_imply_config(configs, args.add_imply, imply_flags, args.skip_added)
Simon Glassc6e73cf2017-06-01 19:39:03 -06002026 return
2027
Simon Glassd9c1da22021-12-18 14:54:31 -07002028 if args.find:
Simon Glass0082b2e2021-12-18 08:09:46 -07002029 do_find_config(configs)
2030 return
2031
Simon Glass43cf08f2017-06-01 19:39:02 -06002032 config_db = {}
Simon Glass1f701862019-10-31 07:42:57 -06002033 db_queue = queue.Queue()
Simon Glass43cf08f2017-06-01 19:39:02 -06002034 t = DatabaseThread(config_db, db_queue)
2035 t.setDaemon(True)
2036 t.start()
2037
Simon Glassd9c1da22021-12-18 14:54:31 -07002038 if not args.cleanup_headers_only:
Masahiro Yamadad0a9d2a2016-07-25 19:15:23 +09002039 check_clean_directory()
Simon Glass06b83a52023-07-19 17:49:05 -06002040 bsettings.setup('')
Simon Glass257f5232017-07-10 14:47:47 -06002041 toolchains = toolchain.Toolchains()
2042 toolchains.GetSettings()
2043 toolchains.Scan(verbose=False)
Simon Glassd9c1da22021-12-18 14:54:31 -07002044 move_config(toolchains, configs, args, db_queue)
Simon Glass43cf08f2017-06-01 19:39:02 -06002045 db_queue.join()
Joe Hershberger23475932015-05-19 13:21:20 -05002046
Masahiro Yamada9566abd2016-05-19 15:52:09 +09002047 if configs:
Simon Glassd9c1da22021-12-18 14:54:31 -07002048 cleanup_headers(configs, args)
Simon Glassd9c1da22021-12-18 14:54:31 -07002049 cleanup_readme(configs, args)
Masahiro Yamadab6160812015-05-20 11:36:07 +09002050
Simon Glassd9c1da22021-12-18 14:54:31 -07002051 if args.commit:
Simon Glass8bf41c22016-09-12 23:18:21 -06002052 subprocess.call(['git', 'add', '-u'])
2053 if configs:
2054 msg = 'Convert %s %sto Kconfig' % (configs[0],
2055 'et al ' if len(configs) > 1 else '')
2056 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' %
2057 '\n '.join(configs))
2058 else:
2059 msg = 'configs: Resync with savedefconfig'
2060 msg += '\n\nRsync all defconfig files using moveconfig.py'
2061 subprocess.call(['git', 'commit', '-s', '-m', msg])
2062
Simon Glassd9c1da22021-12-18 14:54:31 -07002063 if args.build_db:
Simon Glassb3464eb2021-12-18 14:54:35 -07002064 with open(CONFIG_DATABASE, 'w', encoding='utf-8') as fd:
Simon Glass1f701862019-10-31 07:42:57 -06002065 for defconfig, configs in config_db.items():
Simon Glass1c879312017-08-13 16:02:54 -06002066 fd.write('%s\n' % defconfig)
Simon Glass43cf08f2017-06-01 19:39:02 -06002067 for config in sorted(configs.keys()):
Simon Glass1c879312017-08-13 16:02:54 -06002068 fd.write(' %s=%s\n' % (config, configs[config]))
2069 fd.write('\n')
Simon Glass43cf08f2017-06-01 19:39:02 -06002070
Masahiro Yamadab6160812015-05-20 11:36:07 +09002071if __name__ == '__main__':
Simon Glass0082b2e2021-12-18 08:09:46 -07002072 sys.exit(main())