Tom Rini | 10e4779 | 2018-05-06 17:58:06 -0400 | [diff] [blame] | 1 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 2 | # Copyright (c) 2012 The Chromium OS Authors. |
| 3 | # |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 4 | |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 5 | import re |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 6 | import glob |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 7 | from html.parser import HTMLParser |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 8 | import os |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 9 | import sys |
| 10 | import tempfile |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 11 | import urllib.request, urllib.error, urllib.parse |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 12 | |
| 13 | import bsettings |
| 14 | import command |
Simon Glass | 9f1ba0f | 2016-07-27 20:33:02 -0600 | [diff] [blame] | 15 | import terminal |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 16 | import tools |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 17 | |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 18 | (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 19 | PRIORITY_CALC) = list(range(4)) |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 20 | |
Simon Glass | 48ac42e | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 21 | (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) |
| 22 | |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 23 | # Simple class to collect links from a page |
| 24 | class MyHTMLParser(HTMLParser): |
| 25 | def __init__(self, arch): |
| 26 | """Create a new parser |
| 27 | |
| 28 | After the parser runs, self.links will be set to a list of the links |
| 29 | to .xz archives found in the page, and self.arch_link will be set to |
| 30 | the one for the given architecture (or None if not found). |
| 31 | |
| 32 | Args: |
| 33 | arch: Architecture to search for |
| 34 | """ |
| 35 | HTMLParser.__init__(self) |
| 36 | self.arch_link = None |
| 37 | self.links = [] |
Daniel Schwierzeck | 40261f9 | 2018-05-10 07:15:53 -0400 | [diff] [blame] | 38 | self.re_arch = re.compile('[-_]%s-' % arch) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 39 | |
| 40 | def handle_starttag(self, tag, attrs): |
| 41 | if tag == 'a': |
| 42 | for tag, value in attrs: |
| 43 | if tag == 'href': |
| 44 | if value and value.endswith('.xz'): |
| 45 | self.links.append(value) |
Daniel Schwierzeck | 40261f9 | 2018-05-10 07:15:53 -0400 | [diff] [blame] | 46 | if self.re_arch.search(value): |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 47 | self.arch_link = value |
| 48 | |
| 49 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 50 | class Toolchain: |
| 51 | """A single toolchain |
| 52 | |
| 53 | Public members: |
| 54 | gcc: Full path to C compiler |
| 55 | path: Directory path containing C compiler |
| 56 | cross: Cross compile string, e.g. 'arm-linux-' |
| 57 | arch: Architecture of toolchain as determined from the first |
| 58 | component of the filename. E.g. arm-linux-gcc becomes arm |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 59 | priority: Toolchain priority (0=highest, 20=lowest) |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 60 | override_toolchain: Toolchain to use for sandbox, overriding the normal |
| 61 | one |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 62 | """ |
Simon Glass | c6cdd3e | 2016-03-06 19:45:38 -0700 | [diff] [blame] | 63 | def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 64 | arch=None, override_toolchain=None): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 65 | """Create a new toolchain object. |
| 66 | |
| 67 | Args: |
| 68 | fname: Filename of the gcc component |
| 69 | test: True to run the toolchain to test it |
Simon Glass | d6ece32 | 2016-03-06 19:45:35 -0700 | [diff] [blame] | 70 | verbose: True to print out the information |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 71 | priority: Priority to use for this toolchain, or PRIORITY_CALC to |
| 72 | calculate it |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 73 | """ |
| 74 | self.gcc = fname |
| 75 | self.path = os.path.dirname(fname) |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 76 | self.override_toolchain = override_toolchain |
Simon Glass | 28ed006 | 2014-12-01 17:33:58 -0700 | [diff] [blame] | 77 | |
| 78 | # Find the CROSS_COMPILE prefix to use for U-Boot. For example, |
| 79 | # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. |
| 80 | basename = os.path.basename(fname) |
| 81 | pos = basename.rfind('-') |
| 82 | self.cross = basename[:pos + 1] if pos != -1 else '' |
| 83 | |
| 84 | # The architecture is the first part of the name |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 85 | pos = self.cross.find('-') |
Simon Glass | c6cdd3e | 2016-03-06 19:45:38 -0700 | [diff] [blame] | 86 | if arch: |
| 87 | self.arch = arch |
| 88 | else: |
| 89 | self.arch = self.cross[:pos] if pos != -1 else 'sandbox' |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 90 | if self.arch == 'sandbox' and override_toolchain: |
| 91 | self.gcc = override_toolchain |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 92 | |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 93 | env = self.MakeEnvironment(False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 94 | |
| 95 | # As a basic sanity check, run the C compiler with --version |
| 96 | cmd = [fname, '--version'] |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 97 | if priority == PRIORITY_CALC: |
| 98 | self.priority = self.GetPriority(fname) |
| 99 | else: |
| 100 | self.priority = priority |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 101 | if test: |
Stephen Warren | 288d767 | 2013-10-09 14:28:09 -0600 | [diff] [blame] | 102 | result = command.RunPipe([cmd], capture=True, env=env, |
| 103 | raise_on_error=False) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 104 | self.ok = result.return_code == 0 |
| 105 | if verbose: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 106 | print('Tool chain test: ', end=' ') |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 107 | if self.ok: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 108 | print("OK, arch='%s', priority %d" % (self.arch, |
| 109 | self.priority)) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 110 | else: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 111 | print('BAD') |
| 112 | print('Command: ', cmd) |
| 113 | print(result.stdout) |
| 114 | print(result.stderr) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 115 | else: |
| 116 | self.ok = True |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 117 | |
| 118 | def GetPriority(self, fname): |
| 119 | """Return the priority of the toolchain. |
| 120 | |
| 121 | Toolchains are ranked according to their suitability by their |
| 122 | filename prefix. |
| 123 | |
| 124 | Args: |
| 125 | fname: Filename of toolchain |
| 126 | Returns: |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 127 | Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 128 | """ |
Masahiro Yamada | fa25e1f | 2014-07-07 09:47:45 +0900 | [diff] [blame] | 129 | priority_list = ['-elf', '-unknown-linux-gnu', '-linux', |
Tom Rini | 6abc9c8 | 2017-04-14 19:47:50 -0400 | [diff] [blame] | 130 | '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', |
| 131 | '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', |
| 132 | '-linux-gnueabihf', '-le-linux', '-uclinux'] |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 133 | for prio in range(len(priority_list)): |
| 134 | if priority_list[prio] in fname: |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 135 | return PRIORITY_CALC + prio |
| 136 | return PRIORITY_CALC + prio |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 137 | |
York Sun | fb197a8 | 2016-10-04 14:33:51 -0700 | [diff] [blame] | 138 | def GetWrapper(self, show_warning=True): |
| 139 | """Get toolchain wrapper from the setting file. |
| 140 | """ |
Simon Glass | 4557263 | 2019-01-07 16:44:24 -0700 | [diff] [blame] | 141 | value = '' |
| 142 | for name, value in bsettings.GetItems('toolchain-wrapper'): |
York Sun | fb197a8 | 2016-10-04 14:33:51 -0700 | [diff] [blame] | 143 | if not value: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 144 | print("Warning: Wrapper not found") |
York Sun | fb197a8 | 2016-10-04 14:33:51 -0700 | [diff] [blame] | 145 | if value: |
| 146 | value = value + ' ' |
| 147 | |
| 148 | return value |
| 149 | |
Simon Glass | 48ac42e | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 150 | def GetEnvArgs(self, which): |
| 151 | """Get an environment variable/args value based on the the toolchain |
| 152 | |
| 153 | Args: |
| 154 | which: VAR_... value to get |
| 155 | |
| 156 | Returns: |
| 157 | Value of that environment variable or arguments |
| 158 | """ |
| 159 | wrapper = self.GetWrapper() |
| 160 | if which == VAR_CROSS_COMPILE: |
| 161 | return wrapper + os.path.join(self.path, self.cross) |
| 162 | elif which == VAR_PATH: |
| 163 | return self.path |
| 164 | elif which == VAR_ARCH: |
| 165 | return self.arch |
| 166 | elif which == VAR_MAKE_ARGS: |
| 167 | args = self.MakeArgs() |
| 168 | if args: |
| 169 | return ' '.join(args) |
| 170 | return '' |
| 171 | else: |
| 172 | raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) |
| 173 | |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 174 | def MakeEnvironment(self, full_path): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 175 | """Returns an environment for using the toolchain. |
| 176 | |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 177 | Thie takes the current environment and adds CROSS_COMPILE so that |
Daniel Schwierzeck | c9e5f0c | 2017-06-08 03:07:08 +0200 | [diff] [blame] | 178 | the tool chain will operate correctly. This also disables localized |
| 179 | output and possibly unicode encoded output of all build tools by |
| 180 | adding LC_ALL=C. |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 181 | |
| 182 | Args: |
| 183 | full_path: Return the full path in CROSS_COMPILE and don't set |
| 184 | PATH |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 185 | Returns: |
| 186 | Dict containing the environemnt to use. This is based on the current |
| 187 | environment, with changes as needed to CROSS_COMPILE, PATH and |
| 188 | LC_ALL. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 189 | """ |
| 190 | env = dict(os.environ) |
York Sun | fb197a8 | 2016-10-04 14:33:51 -0700 | [diff] [blame] | 191 | wrapper = self.GetWrapper() |
| 192 | |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 193 | if self.override_toolchain: |
| 194 | # We'll use MakeArgs() to provide this |
| 195 | pass |
| 196 | elif full_path: |
York Sun | fb197a8 | 2016-10-04 14:33:51 -0700 | [diff] [blame] | 197 | env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross) |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 198 | else: |
York Sun | fb197a8 | 2016-10-04 14:33:51 -0700 | [diff] [blame] | 199 | env['CROSS_COMPILE'] = wrapper + self.cross |
Simon Glass | d48a46c | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 200 | env['PATH'] = self.path + ':' + env['PATH'] |
| 201 | |
Daniel Schwierzeck | c9e5f0c | 2017-06-08 03:07:08 +0200 | [diff] [blame] | 202 | env['LC_ALL'] = 'C' |
| 203 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 204 | return env |
| 205 | |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 206 | def MakeArgs(self): |
| 207 | """Create the 'make' arguments for a toolchain |
| 208 | |
| 209 | This is only used when the toolchain is being overridden. Since the |
| 210 | U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the |
| 211 | environment (and MakeEnvironment()) to override these values. This |
| 212 | function returns the arguments to accomplish this. |
| 213 | |
| 214 | Returns: |
| 215 | List of arguments to pass to 'make' |
| 216 | """ |
| 217 | if self.override_toolchain: |
| 218 | return ['HOSTCC=%s' % self.override_toolchain, |
| 219 | 'CC=%s' % self.override_toolchain] |
| 220 | return [] |
| 221 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 222 | |
| 223 | class Toolchains: |
| 224 | """Manage a list of toolchains for building U-Boot |
| 225 | |
| 226 | We select one toolchain for each architecture type |
| 227 | |
| 228 | Public members: |
| 229 | toolchains: Dict of Toolchain objects, keyed by architecture name |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 230 | prefixes: Dict of prefixes to check, keyed by architecture. This can |
| 231 | be a full path and toolchain prefix, for example |
| 232 | {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of |
| 233 | something on the search path, for example |
| 234 | {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 235 | paths: List of paths to check for toolchains (may contain wildcards) |
| 236 | """ |
| 237 | |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 238 | def __init__(self, override_toolchain=None): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 239 | self.toolchains = {} |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 240 | self.prefixes = {} |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 241 | self.paths = [] |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 242 | self.override_toolchain = override_toolchain |
Simon Glass | ed098bb | 2014-09-05 19:00:13 -0600 | [diff] [blame] | 243 | self._make_flags = dict(bsettings.GetItems('make-flags')) |
| 244 | |
Simon Glass | f38a642 | 2016-07-27 20:33:01 -0600 | [diff] [blame] | 245 | def GetPathList(self, show_warning=True): |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 246 | """Get a list of available toolchain paths |
| 247 | |
Simon Glass | f38a642 | 2016-07-27 20:33:01 -0600 | [diff] [blame] | 248 | Args: |
| 249 | show_warning: True to show a warning if there are no tool chains. |
| 250 | |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 251 | Returns: |
| 252 | List of strings, each a path to a toolchain mentioned in the |
| 253 | [toolchain] section of the settings file. |
| 254 | """ |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 255 | toolchains = bsettings.GetItems('toolchain') |
Simon Glass | f38a642 | 2016-07-27 20:33:01 -0600 | [diff] [blame] | 256 | if show_warning and not toolchains: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 257 | print(("Warning: No tool chains. Please run 'buildman " |
Simon Glass | 9f1ba0f | 2016-07-27 20:33:02 -0600 | [diff] [blame] | 258 | "--fetch-arch all' to download all available toolchains, or " |
| 259 | "add a [toolchain] section to your buildman config file " |
| 260 | "%s. See README for details" % |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 261 | bsettings.config_fname)) |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 262 | |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 263 | paths = [] |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 264 | for name, value in toolchains: |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 265 | if '*' in value: |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 266 | paths += glob.glob(value) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 267 | else: |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 268 | paths.append(value) |
| 269 | return paths |
| 270 | |
Simon Glass | f38a642 | 2016-07-27 20:33:01 -0600 | [diff] [blame] | 271 | def GetSettings(self, show_warning=True): |
| 272 | """Get toolchain settings from the settings file. |
| 273 | |
| 274 | Args: |
| 275 | show_warning: True to show a warning if there are no tool chains. |
| 276 | """ |
| 277 | self.prefixes = bsettings.GetItems('toolchain-prefix') |
| 278 | self.paths += self.GetPathList(show_warning) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 279 | |
Simon Glass | c6cdd3e | 2016-03-06 19:45:38 -0700 | [diff] [blame] | 280 | def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, |
| 281 | arch=None): |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 282 | """Add a toolchain to our list |
| 283 | |
| 284 | We select the given toolchain as our preferred one for its |
| 285 | architecture if it is a higher priority than the others. |
| 286 | |
| 287 | Args: |
| 288 | fname: Filename of toolchain's gcc driver |
| 289 | test: True to run the toolchain to test it |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 290 | priority: Priority to use for this toolchain |
Simon Glass | c6cdd3e | 2016-03-06 19:45:38 -0700 | [diff] [blame] | 291 | arch: Toolchain architecture, or None if not known |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 292 | """ |
Simon Glass | f77ca5b | 2019-01-07 16:44:20 -0700 | [diff] [blame] | 293 | toolchain = Toolchain(fname, test, verbose, priority, arch, |
| 294 | self.override_toolchain) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 295 | add_it = toolchain.ok |
| 296 | if toolchain.arch in self.toolchains: |
| 297 | add_it = (toolchain.priority < |
| 298 | self.toolchains[toolchain.arch].priority) |
| 299 | if add_it: |
| 300 | self.toolchains[toolchain.arch] = toolchain |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 301 | elif verbose: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 302 | print(("Toolchain '%s' at priority %d will be ignored because " |
Simon Glass | 7b97129 | 2016-03-06 19:45:37 -0700 | [diff] [blame] | 303 | "another toolchain for arch '%s' has priority %d" % |
| 304 | (toolchain.gcc, toolchain.priority, toolchain.arch, |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 305 | self.toolchains[toolchain.arch].priority))) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 306 | |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 307 | def ScanPath(self, path, verbose): |
| 308 | """Scan a path for a valid toolchain |
| 309 | |
| 310 | Args: |
| 311 | path: Path to scan |
| 312 | verbose: True to print out progress information |
| 313 | Returns: |
| 314 | Filename of C compiler if found, else None |
| 315 | """ |
Albert ARIBAUD | d4f22cf | 2015-02-01 00:12:44 +0100 | [diff] [blame] | 316 | fnames = [] |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 317 | for subdir in ['.', 'bin', 'usr/bin']: |
| 318 | dirname = os.path.join(path, subdir) |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 319 | if verbose: print(" - looking in '%s'" % dirname) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 320 | for fname in glob.glob(dirname + '/*gcc'): |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 321 | if verbose: print(" - found '%s'" % fname) |
Albert ARIBAUD | d4f22cf | 2015-02-01 00:12:44 +0100 | [diff] [blame] | 322 | fnames.append(fname) |
| 323 | return fnames |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 324 | |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 325 | def ScanPathEnv(self, fname): |
| 326 | """Scan the PATH environment variable for a given filename. |
| 327 | |
| 328 | Args: |
| 329 | fname: Filename to scan for |
| 330 | Returns: |
| 331 | List of matching pathanames, or [] if none |
| 332 | """ |
| 333 | pathname_list = [] |
| 334 | for path in os.environ["PATH"].split(os.pathsep): |
| 335 | path = path.strip('"') |
| 336 | pathname = os.path.join(path, fname) |
| 337 | if os.path.exists(pathname): |
| 338 | pathname_list.append(pathname) |
| 339 | return pathname_list |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 340 | |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 341 | def Scan(self, verbose): |
| 342 | """Scan for available toolchains and select the best for each arch. |
| 343 | |
| 344 | We look for all the toolchains we can file, figure out the |
| 345 | architecture for each, and whether it works. Then we select the |
| 346 | highest priority toolchain for each arch. |
| 347 | |
| 348 | Args: |
| 349 | verbose: True to print out progress information |
| 350 | """ |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 351 | if verbose: print('Scanning for tool chains') |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 352 | for name, value in self.prefixes: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 353 | if verbose: print(" - scanning prefix '%s'" % value) |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 354 | if os.path.exists(value): |
| 355 | self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) |
| 356 | continue |
| 357 | fname = value + 'gcc' |
| 358 | if os.path.exists(fname): |
| 359 | self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) |
| 360 | continue |
| 361 | fname_list = self.ScanPathEnv(fname) |
| 362 | for f in fname_list: |
| 363 | self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) |
| 364 | if not fname_list: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 365 | raise ValueError("No tool chain found for prefix '%s'" % |
Simon Glass | f590273 | 2016-03-12 18:50:32 -0700 | [diff] [blame] | 366 | value) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 367 | for path in self.paths: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 368 | if verbose: print(" - scanning path '%s'" % path) |
Albert ARIBAUD | d4f22cf | 2015-02-01 00:12:44 +0100 | [diff] [blame] | 369 | fnames = self.ScanPath(path, verbose) |
| 370 | for fname in fnames: |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 371 | self.Add(fname, True, verbose) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 372 | |
| 373 | def List(self): |
| 374 | """List out the selected toolchains for each architecture""" |
Simon Glass | 9f1ba0f | 2016-07-27 20:33:02 -0600 | [diff] [blame] | 375 | col = terminal.Color() |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 376 | print(col.Color(col.BLUE, 'List of available toolchains (%d):' % |
| 377 | len(self.toolchains))) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 378 | if len(self.toolchains): |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 379 | for key, value in sorted(self.toolchains.items()): |
| 380 | print('%-10s: %s' % (key, value.gcc)) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 381 | else: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 382 | print('None') |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 383 | |
| 384 | def Select(self, arch): |
| 385 | """Returns the toolchain for a given architecture |
| 386 | |
| 387 | Args: |
| 388 | args: Name of architecture (e.g. 'arm', 'ppc_8xx') |
| 389 | |
| 390 | returns: |
| 391 | toolchain object, or None if none found |
| 392 | """ |
Simon Glass | c1528c1 | 2014-12-01 17:34:05 -0700 | [diff] [blame] | 393 | for tag, value in bsettings.GetItems('toolchain-alias'): |
| 394 | if arch == tag: |
| 395 | for alias in value.split(): |
| 396 | if alias in self.toolchains: |
| 397 | return self.toolchains[alias] |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 398 | |
| 399 | if not arch in self.toolchains: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 400 | raise ValueError("No tool chain found for arch '%s'" % arch) |
Simon Glass | c05694f | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 401 | return self.toolchains[arch] |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 402 | |
| 403 | def ResolveReferences(self, var_dict, args): |
| 404 | """Resolve variable references in a string |
| 405 | |
| 406 | This converts ${blah} within the string to the value of blah. |
| 407 | This function works recursively. |
| 408 | |
| 409 | Args: |
| 410 | var_dict: Dictionary containing variables and their values |
| 411 | args: String containing make arguments |
| 412 | Returns: |
| 413 | Resolved string |
| 414 | |
| 415 | >>> bsettings.Setup() |
| 416 | >>> tcs = Toolchains() |
| 417 | >>> tcs.Add('fred', False) |
| 418 | >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ |
| 419 | 'second' : '2nd'} |
| 420 | >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') |
| 421 | 'this=OBLIQUE_set' |
| 422 | >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') |
| 423 | 'this=OBLIQUE_setfi2ndrstnd' |
| 424 | """ |
Simon Glass | 53e189d | 2014-08-28 09:43:40 -0600 | [diff] [blame] | 425 | re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 426 | |
| 427 | while True: |
| 428 | m = re_var.search(args) |
| 429 | if not m: |
| 430 | break |
| 431 | lookup = m.group(0)[2:-1] |
| 432 | value = var_dict.get(lookup, '') |
| 433 | args = args[:m.start(0)] + value + args[m.end(0):] |
| 434 | return args |
| 435 | |
| 436 | def GetMakeArguments(self, board): |
| 437 | """Returns 'make' arguments for a given board |
| 438 | |
| 439 | The flags are in a section called 'make-flags'. Flags are named |
| 440 | after the target they represent, for example snapper9260=TESTING=1 |
| 441 | will pass TESTING=1 to make when building the snapper9260 board. |
| 442 | |
| 443 | References to other boards can be added in the string also. For |
| 444 | example: |
| 445 | |
| 446 | [make-flags] |
| 447 | at91-boards=ENABLE_AT91_TEST=1 |
| 448 | snapper9260=${at91-boards} BUILD_TAG=442 |
| 449 | snapper9g45=${at91-boards} BUILD_TAG=443 |
| 450 | |
| 451 | This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 |
| 452 | and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. |
| 453 | |
| 454 | A special 'target' variable is set to the board target. |
| 455 | |
| 456 | Args: |
| 457 | board: Board object for the board to check. |
| 458 | Returns: |
| 459 | 'make' flags for that board, or '' if none |
| 460 | """ |
| 461 | self._make_flags['target'] = board.target |
| 462 | arg_str = self.ResolveReferences(self._make_flags, |
| 463 | self._make_flags.get(board.target, '')) |
Cristian Ciocaltea | 9da17fb | 2019-11-24 22:30:26 +0200 | [diff] [blame] | 464 | args = re.findall("(?:\".*?\"|\S)+", arg_str) |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 465 | i = 0 |
| 466 | while i < len(args): |
Cristian Ciocaltea | 9da17fb | 2019-11-24 22:30:26 +0200 | [diff] [blame] | 467 | args[i] = args[i].replace('"', '') |
Simon Glass | cc246fb | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 468 | if not args[i]: |
| 469 | del args[i] |
| 470 | else: |
| 471 | i += 1 |
| 472 | return args |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 473 | |
| 474 | def LocateArchUrl(self, fetch_arch): |
| 475 | """Find a toolchain available online |
| 476 | |
| 477 | Look in standard places for available toolchains. At present the |
| 478 | only standard place is at kernel.org. |
| 479 | |
| 480 | Args: |
| 481 | arch: Architecture to look for, or 'list' for all |
| 482 | Returns: |
| 483 | If fetch_arch is 'list', a tuple: |
| 484 | Machine architecture (e.g. x86_64) |
| 485 | List of toolchains |
| 486 | else |
| 487 | URL containing this toolchain, if avaialble, else None |
| 488 | """ |
| 489 | arch = command.OutputOneLine('uname', '-m') |
Matthias Brugger | fb50a26 | 2020-01-17 10:53:37 +0100 | [diff] [blame] | 490 | if arch == 'aarch64': |
| 491 | arch = 'arm64' |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 492 | base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' |
Daniel Schwierzeck | 40261f9 | 2018-05-10 07:15:53 -0400 | [diff] [blame] | 493 | versions = ['7.3.0', '6.4.0', '4.9.4'] |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 494 | links = [] |
| 495 | for version in versions: |
| 496 | url = '%s/%s/%s/' % (base, arch, version) |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 497 | print('Checking: %s' % url) |
| 498 | response = urllib.request.urlopen(url) |
| 499 | html = tools.ToString(response.read()) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 500 | parser = MyHTMLParser(fetch_arch) |
| 501 | parser.feed(html) |
| 502 | if fetch_arch == 'list': |
| 503 | links += parser.links |
| 504 | elif parser.arch_link: |
| 505 | return url + parser.arch_link |
| 506 | if fetch_arch == 'list': |
| 507 | return arch, links |
| 508 | return None |
| 509 | |
| 510 | def Download(self, url): |
| 511 | """Download a file to a temporary directory |
| 512 | |
| 513 | Args: |
| 514 | url: URL to download |
| 515 | Returns: |
| 516 | Tuple: |
| 517 | Temporary directory name |
| 518 | Full path to the downloaded archive file in that directory, |
| 519 | or None if there was an error while downloading |
| 520 | """ |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 521 | print('Downloading: %s' % url) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 522 | leaf = url.split('/')[-1] |
| 523 | tmpdir = tempfile.mkdtemp('.buildman') |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 524 | response = urllib.request.urlopen(url) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 525 | fname = os.path.join(tmpdir, leaf) |
| 526 | fd = open(fname, 'wb') |
| 527 | meta = response.info() |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 528 | size = int(meta.get('Content-Length')) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 529 | done = 0 |
| 530 | block_size = 1 << 16 |
| 531 | status = '' |
| 532 | |
| 533 | # Read the file in chunks and show progress as we go |
| 534 | while True: |
| 535 | buffer = response.read(block_size) |
| 536 | if not buffer: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 537 | print(chr(8) * (len(status) + 1), '\r', end=' ') |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 538 | break |
| 539 | |
| 540 | done += len(buffer) |
| 541 | fd.write(buffer) |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 542 | status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, |
| 543 | done * 100 // size) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 544 | status = status + chr(8) * (len(status) + 1) |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 545 | print(status, end=' ') |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 546 | sys.stdout.flush() |
| 547 | fd.close() |
| 548 | if done != size: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 549 | print('Error, failed to download') |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 550 | os.remove(fname) |
| 551 | fname = None |
| 552 | return tmpdir, fname |
| 553 | |
| 554 | def Unpack(self, fname, dest): |
| 555 | """Unpack a tar file |
| 556 | |
| 557 | Args: |
| 558 | fname: Filename to unpack |
| 559 | dest: Destination directory |
| 560 | Returns: |
| 561 | Directory name of the first entry in the archive, without the |
| 562 | trailing / |
| 563 | """ |
| 564 | stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) |
Trevor Woerner | e0557a7 | 2018-11-21 03:31:12 -0500 | [diff] [blame] | 565 | dirs = stdout.splitlines()[1].split('/')[:2] |
| 566 | return '/'.join(dirs) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 567 | |
| 568 | def TestSettingsHasPath(self, path): |
Simon Glass | 43ea128 | 2016-07-27 20:33:03 -0600 | [diff] [blame] | 569 | """Check if buildman will find this toolchain |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 570 | |
| 571 | Returns: |
| 572 | True if the path is in settings, False if not |
| 573 | """ |
Simon Glass | f38a642 | 2016-07-27 20:33:01 -0600 | [diff] [blame] | 574 | paths = self.GetPathList(False) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 575 | return path in paths |
| 576 | |
| 577 | def ListArchs(self): |
| 578 | """List architectures with available toolchains to download""" |
| 579 | host_arch, archives = self.LocateArchUrl('list') |
Trevor Woerner | 10a5162 | 2018-11-21 03:31:13 -0500 | [diff] [blame] | 580 | re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 581 | arch_set = set() |
| 582 | for archive in archives: |
| 583 | # Remove the host architecture from the start |
| 584 | arch = re_arch.match(archive[len(host_arch):]) |
| 585 | if arch: |
Trevor Woerner | 10a5162 | 2018-11-21 03:31:13 -0500 | [diff] [blame] | 586 | if arch.group(1) != '2.0' and arch.group(1) != '64': |
| 587 | arch_set.add(arch.group(1)) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 588 | return sorted(arch_set) |
| 589 | |
| 590 | def FetchAndInstall(self, arch): |
| 591 | """Fetch and install a new toolchain |
| 592 | |
| 593 | arch: |
| 594 | Architecture to fetch, or 'list' to list |
| 595 | """ |
| 596 | # Fist get the URL for this architecture |
Simon Glass | 9f1ba0f | 2016-07-27 20:33:02 -0600 | [diff] [blame] | 597 | col = terminal.Color() |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 598 | print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 599 | url = self.LocateArchUrl(arch) |
| 600 | if not url: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 601 | print(("Cannot find toolchain for arch '%s' - use 'list' to list" % |
| 602 | arch)) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 603 | return 2 |
| 604 | home = os.environ['HOME'] |
| 605 | dest = os.path.join(home, '.buildman-toolchains') |
| 606 | if not os.path.exists(dest): |
| 607 | os.mkdir(dest) |
| 608 | |
| 609 | # Download the tar file for this toolchain and unpack it |
| 610 | tmpdir, tarfile = self.Download(url) |
| 611 | if not tarfile: |
| 612 | return 1 |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 613 | print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ') |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 614 | sys.stdout.flush() |
| 615 | path = self.Unpack(tarfile, dest) |
| 616 | os.remove(tarfile) |
| 617 | os.rmdir(tmpdir) |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 618 | print() |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 619 | |
| 620 | # Check that the toolchain works |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 621 | print(col.Color(col.GREEN, 'Testing')) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 622 | dirpath = os.path.join(dest, path) |
Simon Glass | f675750 | 2015-03-02 17:05:15 -0700 | [diff] [blame] | 623 | compiler_fname_list = self.ScanPath(dirpath, True) |
| 624 | if not compiler_fname_list: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 625 | print('Could not locate C compiler - fetch failed.') |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 626 | return 1 |
Simon Glass | f675750 | 2015-03-02 17:05:15 -0700 | [diff] [blame] | 627 | if len(compiler_fname_list) != 1: |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 628 | print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % |
| 629 | ', '.join(compiler_fname_list))) |
Simon Glass | f675750 | 2015-03-02 17:05:15 -0700 | [diff] [blame] | 630 | toolchain = Toolchain(compiler_fname_list[0], True, True) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 631 | |
| 632 | # Make sure that it will be found by buildman |
| 633 | if not self.TestSettingsHasPath(dirpath): |
Simon Glass | c78ed66 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 634 | print(("Adding 'download' to config file '%s'" % |
| 635 | bsettings.config_fname)) |
Simon Glass | f90e291 | 2016-07-27 20:33:05 -0600 | [diff] [blame] | 636 | bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) |
Simon Glass | 7e803e1 | 2014-12-01 17:34:06 -0700 | [diff] [blame] | 637 | return 0 |