blob: a65737fdf84b586e04791b8c9e8b5c507c68effe [file] [log] [blame]
# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2012 The Chromium OS Authors.
import re
import glob
from HTMLParser import HTMLParser
import os
import sys
import tempfile
import urllib2
import bsettings
import command
import terminal
PRIORITY_CALC) = range(4)
# Simple class to collect links from a page
class MyHTMLParser(HTMLParser):
def __init__(self, arch):
"""Create a new parser
After the parser runs, self.links will be set to a list of the links
to .xz archives found in the page, and self.arch_link will be set to
the one for the given architecture (or None if not found).
arch: Architecture to search for
self.arch_link = None
self.links = []
self.re_arch = re.compile('[-_]%s-' % arch)
def handle_starttag(self, tag, attrs):
if tag == 'a':
for tag, value in attrs:
if tag == 'href':
if value and value.endswith('.xz'):
self.arch_link = value
class Toolchain:
"""A single toolchain
Public members:
gcc: Full path to C compiler
path: Directory path containing C compiler
cross: Cross compile string, e.g. 'arm-linux-'
arch: Architecture of toolchain as determined from the first
component of the filename. E.g. arm-linux-gcc becomes arm
priority: Toolchain priority (0=highest, 20=lowest)
override_toolchain: Toolchain to use for sandbox, overriding the normal
def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
arch=None, override_toolchain=None):
"""Create a new toolchain object.
fname: Filename of the gcc component
test: True to run the toolchain to test it
verbose: True to print out the information
priority: Priority to use for this toolchain, or PRIORITY_CALC to
calculate it
self.gcc = fname
self.path = os.path.dirname(fname)
self.override_toolchain = override_toolchain
# Find the CROSS_COMPILE prefix to use for U-Boot. For example,
# 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
basename = os.path.basename(fname)
pos = basename.rfind('-')
self.cross = basename[:pos + 1] if pos != -1 else ''
# The architecture is the first part of the name
pos = self.cross.find('-')
if arch:
self.arch = arch
self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
if self.arch == 'sandbox' and override_toolchain:
self.gcc = override_toolchain
env = self.MakeEnvironment(False)
# As a basic sanity check, run the C compiler with --version
cmd = [fname, '--version']
if priority == PRIORITY_CALC:
self.priority = self.GetPriority(fname)
self.priority = priority
if test:
result = command.RunPipe([cmd], capture=True, env=env,
self.ok = result.return_code == 0
if verbose:
print 'Tool chain test: ',
if self.ok:
print "OK, arch='%s', priority %d" % (self.arch,
print 'BAD'
print 'Command: ', cmd
print result.stdout
print result.stderr
self.ok = True
def GetPriority(self, fname):
"""Return the priority of the toolchain.
Toolchains are ranked according to their suitability by their
filename prefix.
fname: Filename of toolchain
Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
'-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
'-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
'-linux-gnueabihf', '-le-linux', '-uclinux']
for prio in range(len(priority_list)):
if priority_list[prio] in fname:
return PRIORITY_CALC + prio
return PRIORITY_CALC + prio
def GetWrapper(self, show_warning=True):
"""Get toolchain wrapper from the setting file.
value = ''
for name, value in bsettings.GetItems('toolchain-wrapper'):
if not value:
print "Warning: Wrapper not found"
if value:
value = value + ' '
return value
def MakeEnvironment(self, full_path):
"""Returns an environment for using the toolchain.
Thie takes the current environment and adds CROSS_COMPILE so that
the tool chain will operate correctly. This also disables localized
output and possibly unicode encoded output of all build tools by
adding LC_ALL=C.
full_path: Return the full path in CROSS_COMPILE and don't set
Dict containing the environemnt to use. This is based on the current
environment, with changes as needed to CROSS_COMPILE, PATH and
env = dict(os.environ)
wrapper = self.GetWrapper()
if self.override_toolchain:
# We'll use MakeArgs() to provide this
elif full_path:
env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
env['CROSS_COMPILE'] = wrapper + self.cross
env['PATH'] = self.path + ':' + env['PATH']
env['LC_ALL'] = 'C'
return env
def MakeArgs(self):
"""Create the 'make' arguments for a toolchain
This is only used when the toolchain is being overridden. Since the
U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
environment (and MakeEnvironment()) to override these values. This
function returns the arguments to accomplish this.
List of arguments to pass to 'make'
if self.override_toolchain:
return ['HOSTCC=%s' % self.override_toolchain,
'CC=%s' % self.override_toolchain]
return []
class Toolchains:
"""Manage a list of toolchains for building U-Boot
We select one toolchain for each architecture type
Public members:
toolchains: Dict of Toolchain objects, keyed by architecture name
prefixes: Dict of prefixes to check, keyed by architecture. This can
be a full path and toolchain prefix, for example
{'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
something on the search path, for example
{'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
paths: List of paths to check for toolchains (may contain wildcards)
def __init__(self, override_toolchain=None):
self.toolchains = {}
self.prefixes = {}
self.paths = []
self.override_toolchain = override_toolchain
self._make_flags = dict(bsettings.GetItems('make-flags'))
def GetPathList(self, show_warning=True):
"""Get a list of available toolchain paths
show_warning: True to show a warning if there are no tool chains.
List of strings, each a path to a toolchain mentioned in the
[toolchain] section of the settings file.
toolchains = bsettings.GetItems('toolchain')
if show_warning and not toolchains:
print ("Warning: No tool chains. Please run 'buildman "
"--fetch-arch all' to download all available toolchains, or "
"add a [toolchain] section to your buildman config file "
"%s. See README for details" %
paths = []
for name, value in toolchains:
if '*' in value:
paths += glob.glob(value)
return paths
def GetSettings(self, show_warning=True):
"""Get toolchain settings from the settings file.
show_warning: True to show a warning if there are no tool chains.
self.prefixes = bsettings.GetItems('toolchain-prefix')
self.paths += self.GetPathList(show_warning)
def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
"""Add a toolchain to our list
We select the given toolchain as our preferred one for its
architecture if it is a higher priority than the others.
fname: Filename of toolchain's gcc driver
test: True to run the toolchain to test it
priority: Priority to use for this toolchain
arch: Toolchain architecture, or None if not known
toolchain = Toolchain(fname, test, verbose, priority, arch,
add_it = toolchain.ok
if toolchain.arch in self.toolchains:
add_it = (toolchain.priority <
if add_it:
self.toolchains[toolchain.arch] = toolchain
elif verbose:
print ("Toolchain '%s' at priority %d will be ignored because "
"another toolchain for arch '%s' has priority %d" %
(toolchain.gcc, toolchain.priority, toolchain.arch,
def ScanPath(self, path, verbose):
"""Scan a path for a valid toolchain
path: Path to scan
verbose: True to print out progress information
Filename of C compiler if found, else None
fnames = []
for subdir in ['.', 'bin', 'usr/bin']:
dirname = os.path.join(path, subdir)
if verbose: print " - looking in '%s'" % dirname
for fname in glob.glob(dirname + '/*gcc'):
if verbose: print " - found '%s'" % fname
return fnames
def ScanPathEnv(self, fname):
"""Scan the PATH environment variable for a given filename.
fname: Filename to scan for
List of matching pathanames, or [] if none
pathname_list = []
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
pathname = os.path.join(path, fname)
if os.path.exists(pathname):
return pathname_list
def Scan(self, verbose):
"""Scan for available toolchains and select the best for each arch.
We look for all the toolchains we can file, figure out the
architecture for each, and whether it works. Then we select the
highest priority toolchain for each arch.
verbose: True to print out progress information
if verbose: print 'Scanning for tool chains'
for name, value in self.prefixes:
if verbose: print " - scanning prefix '%s'" % value
if os.path.exists(value):
self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
fname = value + 'gcc'
if os.path.exists(fname):
self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
fname_list = self.ScanPathEnv(fname)
for f in fname_list:
self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
if not fname_list:
raise ValueError, ("No tool chain found for prefix '%s'" %
for path in self.paths:
if verbose: print " - scanning path '%s'" % path
fnames = self.ScanPath(path, verbose)
for fname in fnames:
self.Add(fname, True, verbose)
def List(self):
"""List out the selected toolchains for each architecture"""
col = terminal.Color()
print col.Color(col.BLUE, 'List of available toolchains (%d):' %
if len(self.toolchains):
for key, value in sorted(self.toolchains.iteritems()):
print '%-10s: %s' % (key, value.gcc)
print 'None'
def Select(self, arch):
"""Returns the toolchain for a given architecture
args: Name of architecture (e.g. 'arm', 'ppc_8xx')
toolchain object, or None if none found
for tag, value in bsettings.GetItems('toolchain-alias'):
if arch == tag:
for alias in value.split():
if alias in self.toolchains:
return self.toolchains[alias]
if not arch in self.toolchains:
raise ValueError, ("No tool chain found for arch '%s'" % arch)
return self.toolchains[arch]
def ResolveReferences(self, var_dict, args):
"""Resolve variable references in a string
This converts ${blah} within the string to the value of blah.
This function works recursively.
var_dict: Dictionary containing variables and their values
args: String containing make arguments
Resolved string
>>> bsettings.Setup()
>>> tcs = Toolchains()
>>> tcs.Add('fred', False)
>>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
'second' : '2nd'}
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
while True:
m =
if not m:
lookup =[2:-1]
value = var_dict.get(lookup, '')
args = args[:m.start(0)] + value + args[m.end(0):]
return args
def GetMakeArguments(self, board):
"""Returns 'make' arguments for a given board
The flags are in a section called 'make-flags'. Flags are named
after the target they represent, for example snapper9260=TESTING=1
will pass TESTING=1 to make when building the snapper9260 board.
References to other boards can be added in the string also. For
snapper9260=${at91-boards} BUILD_TAG=442
snapper9g45=${at91-boards} BUILD_TAG=443
This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
A special 'target' variable is set to the board target.
board: Board object for the board to check.
'make' flags for that board, or '' if none
self._make_flags['target'] =
arg_str = self.ResolveReferences(self._make_flags,
self._make_flags.get(, ''))
args = arg_str.split(' ')
i = 0
while i < len(args):
if not args[i]:
del args[i]
i += 1
return args
def LocateArchUrl(self, fetch_arch):
"""Find a toolchain available online
Look in standard places for available toolchains. At present the
only standard place is at
arch: Architecture to look for, or 'list' for all
If fetch_arch is 'list', a tuple:
Machine architecture (e.g. x86_64)
List of toolchains
URL containing this toolchain, if avaialble, else None
arch = command.OutputOneLine('uname', '-m')
base = ''
versions = ['7.3.0', '6.4.0', '4.9.4']
links = []
for version in versions:
url = '%s/%s/%s/' % (base, arch, version)
print 'Checking: %s' % url
response = urllib2.urlopen(url)
html =
parser = MyHTMLParser(fetch_arch)
if fetch_arch == 'list':
links += parser.links
elif parser.arch_link:
return url + parser.arch_link
if fetch_arch == 'list':
return arch, links
return None
def Download(self, url):
"""Download a file to a temporary directory
url: URL to download
Temporary directory name
Full path to the downloaded archive file in that directory,
or None if there was an error while downloading
print 'Downloading: %s' % url
leaf = url.split('/')[-1]
tmpdir = tempfile.mkdtemp('.buildman')
response = urllib2.urlopen(url)
fname = os.path.join(tmpdir, leaf)
fd = open(fname, 'wb')
meta =
size = int(meta.getheaders('Content-Length')[0])
done = 0
block_size = 1 << 16
status = ''
# Read the file in chunks and show progress as we go
while True:
buffer =
if not buffer:
print chr(8) * (len(status) + 1), '\r',
done += len(buffer)
status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
done * 100 / size)
status = status + chr(8) * (len(status) + 1)
print status,
if done != size:
print 'Error, failed to download'
fname = None
return tmpdir, fname
def Unpack(self, fname, dest):
"""Unpack a tar file
fname: Filename to unpack
dest: Destination directory
Directory name of the first entry in the archive, without the
trailing /
stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
dirs = stdout.splitlines()[1].split('/')[:2]
return '/'.join(dirs)
def TestSettingsHasPath(self, path):
"""Check if buildman will find this toolchain
True if the path is in settings, False if not
paths = self.GetPathList(False)
return path in paths
def ListArchs(self):
"""List architectures with available toolchains to download"""
host_arch, archives = self.LocateArchUrl('list')
re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
arch_set = set()
for archive in archives:
# Remove the host architecture from the start
arch = re_arch.match(archive[len(host_arch):])
if arch:
if != '2.0' and != '64':
return sorted(arch_set)
def FetchAndInstall(self, arch):
"""Fetch and install a new toolchain
Architecture to fetch, or 'list' to list
# Fist get the URL for this architecture
col = terminal.Color()
print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
url = self.LocateArchUrl(arch)
if not url:
print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
return 2
home = os.environ['HOME']
dest = os.path.join(home, '.buildman-toolchains')
if not os.path.exists(dest):
# Download the tar file for this toolchain and unpack it
tmpdir, tarfile = self.Download(url)
if not tarfile:
return 1
print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
path = self.Unpack(tarfile, dest)
# Check that the toolchain works
print col.Color(col.GREEN, 'Testing')
dirpath = os.path.join(dest, path)
compiler_fname_list = self.ScanPath(dirpath, True)
if not compiler_fname_list:
print 'Could not locate C compiler - fetch failed.'
return 1
if len(compiler_fname_list) != 1:
print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
', '.join(compiler_fname_list))
toolchain = Toolchain(compiler_fname_list[0], True, True)
# Make sure that it will be found by buildman
if not self.TestSettingsHasPath(dirpath):
print ("Adding 'download' to config file '%s'" %
bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
return 0