tools/buildman/toolchain.py

   1 # Copyright (c) 2012 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import re
   7 import glob
   8 from HTMLParser import HTMLParser
   9 import os
  10 import sys
  11 import tempfile
  12 import urllib2
  13
  14 import bsettings
  15 import command
  16 import terminal
  17
  18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
  19     PRIORITY_CALC) = range(4)
  20
  21 # Simple class to collect links from a page
  22 class MyHTMLParser(HTMLParser):
  23     def __init__(self, arch):
  24         """Create a new parser
  25
  26         After the parser runs, self.links will be set to a list of the links
  27         to .xz archives found in the page, and self.arch_link will be set to
  28         the one for the given architecture (or None if not found).
  29
  30         Args:
  31             arch: Architecture to search for
  32         """
  33         HTMLParser.__init__(self)
  34         self.arch_link = None
  35         self.links = []
  36         self._match = '_%s-' % arch
  37
  38     def handle_starttag(self, tag, attrs):
  39         if tag == 'a':
  40             for tag, value in attrs:
  41                 if tag == 'href':
  42                     if value and value.endswith('.xz'):
  43                         self.links.append(value)
  44                         if self._match in value:
  45                             self.arch_link = value
  46
  47
  48 class Toolchain:
  49     """A single toolchain
  50
  51     Public members:
  52         gcc: Full path to C compiler
  53         path: Directory path containing C compiler
  54         cross: Cross compile string, e.g. 'arm-linux-'
  55         arch: Architecture of toolchain as determined from the first
  56                 component of the filename. E.g. arm-linux-gcc becomes arm
  57         priority: Toolchain priority (0=highest, 20=lowest)
  58     """
  59     def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
  60                  arch=None):
  61         """Create a new toolchain object.
  62
  63         Args:
  64             fname: Filename of the gcc component
  65             test: True to run the toolchain to test it
  66             verbose: True to print out the information
  67             priority: Priority to use for this toolchain, or PRIORITY_CALC to
  68                 calculate it
  69         """
  70         self.gcc = fname
  71         self.path = os.path.dirname(fname)
  72
  73         # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
  74         # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
  75         basename = os.path.basename(fname)
  76         pos = basename.rfind('-')
  77         self.cross = basename[:pos + 1] if pos != -1 else ''
  78
  79         # The architecture is the first part of the name
  80         pos = self.cross.find('-')
  81         if arch:
  82             self.arch = arch
  83         else:
  84             self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
  85
  86         env = self.MakeEnvironment(False)
  87
  88         # As a basic sanity check, run the C compiler with --version
  89         cmd = [fname, '--version']
  90         if priority == PRIORITY_CALC:
  91             self.priority = self.GetPriority(fname)
  92         else:
  93             self.priority = priority
  94         if test:
  95             result = command.RunPipe([cmd], capture=True, env=env,
  96                                      raise_on_error=False)
  97             self.ok = result.return_code == 0
  98             if verbose:
  99                 print 'Tool chain test: ',
 100                 if self.ok:
 101                     print "OK, arch='%s', priority %d" % (self.arch,
 102                                                           self.priority)
 103                 else:
 104                     print 'BAD'
 105                     print 'Command: ', cmd
 106                     print result.stdout
 107                     print result.stderr
 108         else:
 109             self.ok = True
 110
 111     def GetPriority(self, fname):
 112         """Return the priority of the toolchain.
 113
 114         Toolchains are ranked according to their suitability by their
 115         filename prefix.
 116
 117         Args:
 118             fname: Filename of toolchain
 119         Returns:
 120             Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
 121         """
 122         priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
 123             '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
 124             '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
 125             '-linux-gnueabihf', '-le-linux', '-uclinux']
 126         for prio in range(len(priority_list)):
 127             if priority_list[prio] in fname:
 128                 return PRIORITY_CALC + prio
 129         return PRIORITY_CALC + prio
 130
 131     def GetWrapper(self, show_warning=True):
 132         """Get toolchain wrapper from the setting file.
 133         """
 134         value = ''
 135         for name, value in bsettings.GetItems('toolchain-wrapper'):
 136             if not value:
 137                 print "Warning: Wrapper not found"
 138         if value:
 139             value = value + ' '
 140
 141         return value
 142
 143     def MakeEnvironment(self, full_path):
 144         """Returns an environment for using the toolchain.
 145
 146         Thie takes the current environment and adds CROSS_COMPILE so that
 147         the tool chain will operate correctly. This also disables localized
 148         output and possibly unicode encoded output of all build tools by
 149         adding LC_ALL=C.
 150
 151         Args:
 152             full_path: Return the full path in CROSS_COMPILE and don't set
 153                 PATH
 154         """
 155         env = dict(os.environ)
 156         wrapper = self.GetWrapper()
 157
 158         if full_path:
 159             env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
 160         else:
 161             env['CROSS_COMPILE'] = wrapper + self.cross
 162             env['PATH'] = self.path + ':' + env['PATH']
 163
 164         env['LC_ALL'] = 'C'
 165
 166         return env
 167
 168
 169 class Toolchains:
 170     """Manage a list of toolchains for building U-Boot
 171
 172     We select one toolchain for each architecture type
 173
 174     Public members:
 175         toolchains: Dict of Toolchain objects, keyed by architecture name
 176         prefixes: Dict of prefixes to check, keyed by architecture. This can
 177             be a full path and toolchain prefix, for example
 178             {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
 179             something on the search path, for example
 180             {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
 181         paths: List of paths to check for toolchains (may contain wildcards)
 182     """
 183
 184     def __init__(self):
 185         self.toolchains = {}
 186         self.prefixes = {}
 187         self.paths = []
 188         self._make_flags = dict(bsettings.GetItems('make-flags'))
 189
 190     def GetPathList(self, show_warning=True):
 191         """Get a list of available toolchain paths
 192
 193         Args:
 194             show_warning: True to show a warning if there are no tool chains.
 195
 196         Returns:
 197             List of strings, each a path to a toolchain mentioned in the
 198             [toolchain] section of the settings file.
 199         """
 200         toolchains = bsettings.GetItems('toolchain')
 201         if show_warning and not toolchains:
 202             print ("Warning: No tool chains. Please run 'buildman "
 203                    "--fetch-arch all' to download all available toolchains, or "
 204                    "add a [toolchain] section to your buildman config file "
 205                    "%s. See README for details" %
 206                    bsettings.config_fname)
 207
 208         paths = []
 209         for name, value in toolchains:
 210             if '*' in value:
 211                 paths += glob.glob(value)
 212             else:
 213                 paths.append(value)
 214         return paths
 215
 216     def GetSettings(self, show_warning=True):
 217         """Get toolchain settings from the settings file.
 218
 219         Args:
 220             show_warning: True to show a warning if there are no tool chains.
 221         """
 222         self.prefixes = bsettings.GetItems('toolchain-prefix')
 223         self.paths += self.GetPathList(show_warning)
 224
 225     def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
 226             arch=None):
 227         """Add a toolchain to our list
 228
 229         We select the given toolchain as our preferred one for its
 230         architecture if it is a higher priority than the others.
 231
 232         Args:
 233             fname: Filename of toolchain's gcc driver
 234             test: True to run the toolchain to test it
 235             priority: Priority to use for this toolchain
 236             arch: Toolchain architecture, or None if not known
 237         """
 238         toolchain = Toolchain(fname, test, verbose, priority, arch)
 239         add_it = toolchain.ok
 240         if toolchain.arch in self.toolchains:
 241             add_it = (toolchain.priority <
 242                         self.toolchains[toolchain.arch].priority)
 243         if add_it:
 244             self.toolchains[toolchain.arch] = toolchain
 245         elif verbose:
 246             print ("Toolchain '%s' at priority %d will be ignored because "
 247                    "another toolchain for arch '%s' has priority %d" %
 248                    (toolchain.gcc, toolchain.priority, toolchain.arch,
 249                     self.toolchains[toolchain.arch].priority))
 250
 251     def ScanPath(self, path, verbose):
 252         """Scan a path for a valid toolchain
 253
 254         Args:
 255             path: Path to scan
 256             verbose: True to print out progress information
 257         Returns:
 258             Filename of C compiler if found, else None
 259         """
 260         fnames = []
 261         for subdir in ['.', 'bin', 'usr/bin']:
 262             dirname = os.path.join(path, subdir)
 263             if verbose: print "      - looking in '%s'" % dirname
 264             for fname in glob.glob(dirname + '/*gcc'):
 265                 if verbose: print "         - found '%s'" % fname
 266                 fnames.append(fname)
 267         return fnames
 268
 269     def ScanPathEnv(self, fname):
 270         """Scan the PATH environment variable for a given filename.
 271
 272         Args:
 273             fname: Filename to scan for
 274         Returns:
 275             List of matching pathanames, or [] if none
 276         """
 277         pathname_list = []
 278         for path in os.environ["PATH"].split(os.pathsep):
 279             path = path.strip('"')
 280             pathname = os.path.join(path, fname)
 281             if os.path.exists(pathname):
 282                 pathname_list.append(pathname)
 283         return pathname_list
 284
 285     def Scan(self, verbose):
 286         """Scan for available toolchains and select the best for each arch.
 287
 288         We look for all the toolchains we can file, figure out the
 289         architecture for each, and whether it works. Then we select the
 290         highest priority toolchain for each arch.
 291
 292         Args:
 293             verbose: True to print out progress information
 294         """
 295         if verbose: print 'Scanning for tool chains'
 296         for name, value in self.prefixes:
 297             if verbose: print "   - scanning prefix '%s'" % value
 298             if os.path.exists(value):
 299                 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
 300                 continue
 301             fname = value + 'gcc'
 302             if os.path.exists(fname):
 303                 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
 304                 continue
 305             fname_list = self.ScanPathEnv(fname)
 306             for f in fname_list:
 307                 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
 308             if not fname_list:
 309                 raise ValueError, ("No tool chain found for prefix '%s'" %
 310                                    value)
 311         for path in self.paths:
 312             if verbose: print "   - scanning path '%s'" % path
 313             fnames = self.ScanPath(path, verbose)
 314             for fname in fnames:
 315                 self.Add(fname, True, verbose)
 316
 317     def List(self):
 318         """List out the selected toolchains for each architecture"""
 319         col = terminal.Color()
 320         print col.Color(col.BLUE, 'List of available toolchains (%d):' %
 321                         len(self.toolchains))
 322         if len(self.toolchains):
 323             for key, value in sorted(self.toolchains.iteritems()):
 324                 print '%-10s: %s' % (key, value.gcc)
 325         else:
 326             print 'None'
 327
 328     def Select(self, arch):
 329         """Returns the toolchain for a given architecture
 330
 331         Args:
 332             args: Name of architecture (e.g. 'arm', 'ppc_8xx')
 333
 334         returns:
 335             toolchain object, or None if none found
 336         """
 337         for tag, value in bsettings.GetItems('toolchain-alias'):
 338             if arch == tag:
 339                 for alias in value.split():
 340                     if alias in self.toolchains:
 341                         return self.toolchains[alias]
 342
 343         if not arch in self.toolchains:
 344             raise ValueError, ("No tool chain found for arch '%s'" % arch)
 345         return self.toolchains[arch]
 346
 347     def ResolveReferences(self, var_dict, args):
 348         """Resolve variable references in a string
 349
 350         This converts ${blah} within the string to the value of blah.
 351         This function works recursively.
 352
 353         Args:
 354             var_dict: Dictionary containing variables and their values
 355             args: String containing make arguments
 356         Returns:
 357             Resolved string
 358
 359         >>> bsettings.Setup()
 360         >>> tcs = Toolchains()
 361         >>> tcs.Add('fred', False)
 362         >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
 363                         'second' : '2nd'}
 364         >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
 365         'this=OBLIQUE_set'
 366         >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
 367         'this=OBLIQUE_setfi2ndrstnd'
 368         """
 369         re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
 370
 371         while True:
 372             m = re_var.search(args)
 373             if not m:
 374                 break
 375             lookup = m.group(0)[2:-1]
 376             value = var_dict.get(lookup, '')
 377             args = args[:m.start(0)] + value + args[m.end(0):]
 378         return args
 379
 380     def GetMakeArguments(self, board):
 381         """Returns 'make' arguments for a given board
 382
 383         The flags are in a section called 'make-flags'. Flags are named
 384         after the target they represent, for example snapper9260=TESTING=1
 385         will pass TESTING=1 to make when building the snapper9260 board.
 386
 387         References to other boards can be added in the string also. For
 388         example:
 389
 390         [make-flags]
 391         at91-boards=ENABLE_AT91_TEST=1
 392         snapper9260=${at91-boards} BUILD_TAG=442
 393         snapper9g45=${at91-boards} BUILD_TAG=443
 394
 395         This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
 396         and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
 397
 398         A special 'target' variable is set to the board target.
 399
 400         Args:
 401             board: Board object for the board to check.
 402         Returns:
 403             'make' flags for that board, or '' if none
 404         """
 405         self._make_flags['target'] = board.target
 406         arg_str = self.ResolveReferences(self._make_flags,
 407                            self._make_flags.get(board.target, ''))
 408         args = arg_str.split(' ')
 409         i = 0
 410         while i < len(args):
 411             if not args[i]:
 412                 del args[i]
 413             else:
 414                 i += 1
 415         return args
 416
 417     def LocateArchUrl(self, fetch_arch):
 418         """Find a toolchain available online
 419
 420         Look in standard places for available toolchains. At present the
 421         only standard place is at kernel.org.
 422
 423         Args:
 424             arch: Architecture to look for, or 'list' for all
 425         Returns:
 426             If fetch_arch is 'list', a tuple:
 427                 Machine architecture (e.g. x86_64)
 428                 List of toolchains
 429             else
 430                 URL containing this toolchain, if avaialble, else None
 431         """
 432         arch = command.OutputOneLine('uname', '-m')
 433         base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
 434         versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
 435         links = []
 436         for version in versions:
 437             url = '%s/%s/%s/' % (base, arch, version)
 438             print 'Checking: %s' % url
 439             response = urllib2.urlopen(url)
 440             html = response.read()
 441             parser = MyHTMLParser(fetch_arch)
 442             parser.feed(html)
 443             if fetch_arch == 'list':
 444                 links += parser.links
 445             elif parser.arch_link:
 446                 return url + parser.arch_link
 447         if fetch_arch == 'list':
 448             return arch, links
 449         return None
 450
 451     def Download(self, url):
 452         """Download a file to a temporary directory
 453
 454         Args:
 455             url: URL to download
 456         Returns:
 457             Tuple:
 458                 Temporary directory name
 459                 Full path to the downloaded archive file in that directory,
 460                     or None if there was an error while downloading
 461         """
 462         print 'Downloading: %s' % url
 463         leaf = url.split('/')[-1]
 464         tmpdir = tempfile.mkdtemp('.buildman')
 465         response = urllib2.urlopen(url)
 466         fname = os.path.join(tmpdir, leaf)
 467         fd = open(fname, 'wb')
 468         meta = response.info()
 469         size = int(meta.getheaders('Content-Length')[0])
 470         done = 0
 471         block_size = 1 << 16
 472         status = ''
 473
 474         # Read the file in chunks and show progress as we go
 475         while True:
 476             buffer = response.read(block_size)
 477             if not buffer:
 478                 print chr(8) * (len(status) + 1), '\r',
 479                 break
 480
 481             done += len(buffer)
 482             fd.write(buffer)
 483             status = r'%10d MiB  [%3d%%]' % (done / 1024 / 1024,
 484                                              done * 100 / size)
 485             status = status + chr(8) * (len(status) + 1)
 486             print status,
 487             sys.stdout.flush()
 488         fd.close()
 489         if done != size:
 490             print 'Error, failed to download'
 491             os.remove(fname)
 492             fname = None
 493         return tmpdir, fname
 494
 495     def Unpack(self, fname, dest):
 496         """Unpack a tar file
 497
 498         Args:
 499             fname: Filename to unpack
 500             dest: Destination directory
 501         Returns:
 502             Directory name of the first entry in the archive, without the
 503             trailing /
 504         """
 505         stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
 506         return stdout.splitlines()[0][:-1]
 507
 508     def TestSettingsHasPath(self, path):
 509         """Check if buildman will find this toolchain
 510
 511         Returns:
 512             True if the path is in settings, False if not
 513         """
 514         paths = self.GetPathList(False)
 515         return path in paths
 516
 517     def ListArchs(self):
 518         """List architectures with available toolchains to download"""
 519         host_arch, archives = self.LocateArchUrl('list')
 520         re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
 521         arch_set = set()
 522         for archive in archives:
 523             # Remove the host architecture from the start
 524             arch = re_arch.match(archive[len(host_arch):])
 525             if arch:
 526                 arch_set.add(arch.group(1))
 527         return sorted(arch_set)
 528
 529     def FetchAndInstall(self, arch):
 530         """Fetch and install a new toolchain
 531
 532         arch:
 533             Architecture to fetch, or 'list' to list
 534         """
 535         # Fist get the URL for this architecture
 536         col = terminal.Color()
 537         print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
 538         url = self.LocateArchUrl(arch)
 539         if not url:
 540             print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
 541                    arch)
 542             return 2
 543         home = os.environ['HOME']
 544         dest = os.path.join(home, '.buildman-toolchains')
 545         if not os.path.exists(dest):
 546             os.mkdir(dest)
 547
 548         # Download the tar file for this toolchain and unpack it
 549         tmpdir, tarfile = self.Download(url)
 550         if not tarfile:
 551             return 1
 552         print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
 553         sys.stdout.flush()
 554         path = self.Unpack(tarfile, dest)
 555         os.remove(tarfile)
 556         os.rmdir(tmpdir)
 557         print
 558
 559         # Check that the toolchain works
 560         print col.Color(col.GREEN, 'Testing')
 561         dirpath = os.path.join(dest, path)
 562         compiler_fname_list = self.ScanPath(dirpath, True)
 563         if not compiler_fname_list:
 564             print 'Could not locate C compiler - fetch failed.'
 565             return 1
 566         if len(compiler_fname_list) != 1:
 567             print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
 568                             ', '.join(compiler_fname_list))
 569         toolchain = Toolchain(compiler_fname_list[0], True, True)
 570
 571         # Make sure that it will be found by buildman
 572         if not self.TestSettingsHasPath(dirpath):
 573             print ("Adding 'download' to config file '%s'" %
 574                    bsettings.config_fname)
 575             bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
 576         return 0