| # Copyright (c) 2013 The Chromium OS Authors. |
| # |
| # SPDX-License-Identifier: GPL-2.0+ |
| # |
| |
| import multiprocessing |
| import os |
| import shutil |
| import sys |
| |
| import board |
| import bsettings |
| from builder import Builder |
| import gitutil |
| import patchstream |
| import terminal |
| from terminal import Print |
| import toolchain |
| import command |
| import subprocess |
| |
| def GetPlural(count): |
| """Returns a plural 's' if count is not 1""" |
| return 's' if count != 1 else '' |
| |
| def GetActionSummary(is_summary, commits, selected, options): |
| """Return a string summarising the intended action. |
| |
| Returns: |
| Summary string. |
| """ |
| if commits: |
| count = len(commits) |
| count = (count + options.step - 1) / options.step |
| commit_str = '%d commit%s' % (count, GetPlural(count)) |
| else: |
| commit_str = 'current source' |
| str = '%s %s for %d boards' % ( |
| 'Summary of' if is_summary else 'Building', commit_str, |
| len(selected)) |
| str += ' (%d thread%s, %d job%s per thread)' % (options.threads, |
| GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) |
| return str |
| |
| def ShowActions(series, why_selected, boards_selected, builder, options): |
| """Display a list of actions that we would take, if not a dry run. |
| |
| Args: |
| series: Series object |
| why_selected: Dictionary where each key is a buildman argument |
| provided by the user, and the value is the list of boards |
| brought in by that argument. For example, 'arm' might bring |
| in 400 boards, so in this case the key would be 'arm' and |
| the value would be a list of board names. |
| boards_selected: Dict of selected boards, key is target name, |
| value is Board object |
| builder: The builder that will be used to build the commits |
| options: Command line options object |
| """ |
| col = terminal.Color() |
| print 'Dry run, so not doing much. But I would do this:' |
| print |
| if series: |
| commits = series.commits |
| else: |
| commits = None |
| print GetActionSummary(False, commits, boards_selected, |
| options) |
| print 'Build directory: %s' % builder.base_dir |
| if commits: |
| for upto in range(0, len(series.commits), options.step): |
| commit = series.commits[upto] |
| print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), |
| print commit.subject |
| print |
| for arg in why_selected: |
| if arg != 'all': |
| print arg, ': %d boards' % len(why_selected[arg]) |
| if options.verbose: |
| print ' %s' % ' '.join(why_selected[arg]) |
| print ('Total boards to build for each commit: %d\n' % |
| len(why_selected['all'])) |
| |
| def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, |
| clean_dir=False): |
| """The main control code for buildman |
| |
| Args: |
| options: Command line options object |
| args: Command line arguments (list of strings) |
| toolchains: Toolchains to use - this should be a Toolchains() |
| object. If None, then it will be created and scanned |
| make_func: Make function to use for the builder. This is called |
| to execute 'make'. If this is None, the normal function |
| will be used, which calls the 'make' tool with suitable |
| arguments. This setting is useful for tests. |
| board: Boards() object to use, containing a list of available |
| boards. If this is None it will be created and scanned. |
| """ |
| global builder |
| |
| if options.full_help: |
| pager = os.getenv('PAGER') |
| if not pager: |
| pager = 'more' |
| fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), |
| 'README') |
| command.Run(pager, fname) |
| return 0 |
| |
| gitutil.Setup() |
| col = terminal.Color() |
| |
| options.git_dir = os.path.join(options.git, '.git') |
| |
| no_toolchains = toolchains is None |
| if no_toolchains: |
| toolchains = toolchain.Toolchains() |
| |
| if options.fetch_arch: |
| if options.fetch_arch == 'list': |
| sorted_list = toolchains.ListArchs() |
| print col.Color(col.BLUE, 'Available architectures: %s\n' % |
| ' '.join(sorted_list)) |
| return 0 |
| else: |
| fetch_arch = options.fetch_arch |
| if fetch_arch == 'all': |
| fetch_arch = ','.join(toolchains.ListArchs()) |
| print col.Color(col.CYAN, '\nDownloading toolchains: %s' % |
| fetch_arch) |
| for arch in fetch_arch.split(','): |
| print |
| ret = toolchains.FetchAndInstall(arch) |
| if ret: |
| return ret |
| return 0 |
| |
| if no_toolchains: |
| toolchains.GetSettings() |
| toolchains.Scan(options.list_tool_chains) |
| if options.list_tool_chains: |
| toolchains.List() |
| print |
| return 0 |
| |
| # Work out how many commits to build. We want to build everything on the |
| # branch. We also build the upstream commit as a control so we can see |
| # problems introduced by the first commit on the branch. |
| count = options.count |
| has_range = options.branch and '..' in options.branch |
| if count == -1: |
| if not options.branch: |
| count = 1 |
| else: |
| if has_range: |
| count, msg = gitutil.CountCommitsInRange(options.git_dir, |
| options.branch) |
| else: |
| count, msg = gitutil.CountCommitsInBranch(options.git_dir, |
| options.branch) |
| if count is None: |
| sys.exit(col.Color(col.RED, msg)) |
| elif count == 0: |
| sys.exit(col.Color(col.RED, "Range '%s' has no commits" % |
| options.branch)) |
| if msg: |
| print col.Color(col.YELLOW, msg) |
| count += 1 # Build upstream commit also |
| |
| if not count: |
| str = ("No commits found to process in branch '%s': " |
| "set branch's upstream or use -c flag" % options.branch) |
| sys.exit(col.Color(col.RED, str)) |
| |
| # Work out what subset of the boards we are building |
| if not boards: |
| board_file = os.path.join(options.git, 'boards.cfg') |
| status = subprocess.call([os.path.join(options.git, |
| 'tools/genboardscfg.py')]) |
| if status != 0: |
| sys.exit("Failed to generate boards.cfg") |
| |
| boards = board.Boards() |
| boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) |
| |
| exclude = [] |
| if options.exclude: |
| for arg in options.exclude: |
| exclude += arg.split(',') |
| |
| why_selected = boards.SelectBoards(args, exclude) |
| selected = boards.GetSelected() |
| if not len(selected): |
| sys.exit(col.Color(col.RED, 'No matching boards found')) |
| |
| # Read the metadata from the commits. First look at the upstream commit, |
| # then the ones in the branch. We would like to do something like |
| # upstream/master~..branch but that isn't possible if upstream/master is |
| # a merge commit (it will list all the commits that form part of the |
| # merge) |
| # Conflicting tags are not a problem for buildman, since it does not use |
| # them. For example, Series-version is not useful for buildman. On the |
| # other hand conflicting tags will cause an error. So allow later tags |
| # to overwrite earlier ones by setting allow_overwrite=True |
| if options.branch: |
| if count == -1: |
| if has_range: |
| range_expr = options.branch |
| else: |
| range_expr = gitutil.GetRangeInBranch(options.git_dir, |
| options.branch) |
| upstream_commit = gitutil.GetUpstream(options.git_dir, |
| options.branch) |
| series = patchstream.GetMetaDataForList(upstream_commit, |
| options.git_dir, 1, series=None, allow_overwrite=True) |
| |
| series = patchstream.GetMetaDataForList(range_expr, |
| options.git_dir, None, series, allow_overwrite=True) |
| else: |
| # Honour the count |
| series = patchstream.GetMetaDataForList(options.branch, |
| options.git_dir, count, series=None, allow_overwrite=True) |
| else: |
| series = None |
| if not options.dry_run: |
| options.verbose = True |
| if not options.summary: |
| options.show_errors = True |
| |
| # By default we have one thread per CPU. But if there are not enough jobs |
| # we can have fewer threads and use a high '-j' value for make. |
| if not options.threads: |
| options.threads = min(multiprocessing.cpu_count(), len(selected)) |
| if not options.jobs: |
| options.jobs = max(1, (multiprocessing.cpu_count() + |
| len(selected) - 1) / len(selected)) |
| |
| if not options.step: |
| options.step = len(series.commits) - 1 |
| |
| gnu_make = command.Output(os.path.join(options.git, |
| 'scripts/show-gnu-make'), raise_on_error=False).rstrip() |
| if not gnu_make: |
| sys.exit('GNU Make not found') |
| |
| # Create a new builder with the selected options. |
| output_dir = options.output_dir |
| if options.branch: |
| dirname = options.branch.replace('/', '_') |
| # As a special case allow the board directory to be placed in the |
| # output directory itself rather than any subdirectory. |
| if not options.no_subdirs: |
| output_dir = os.path.join(options.output_dir, dirname) |
| if (clean_dir and output_dir != options.output_dir and |
| os.path.exists(output_dir)): |
| shutil.rmtree(output_dir) |
| builder = Builder(toolchains, output_dir, options.git_dir, |
| options.threads, options.jobs, gnu_make=gnu_make, checkout=True, |
| show_unknown=options.show_unknown, step=options.step, |
| no_subdirs=options.no_subdirs, full_path=options.full_path, |
| verbose_build=options.verbose_build, |
| incremental=options.incremental, |
| per_board_out_dir=options.per_board_out_dir, |
| config_only=options.config_only, |
| squash_config_y=not options.preserve_config_y) |
| builder.force_config_on_failure = not options.quick |
| if make_func: |
| builder.do_make = make_func |
| |
| # For a dry run, just show our actions as a sanity check |
| if options.dry_run: |
| ShowActions(series, why_selected, selected, builder, options) |
| else: |
| builder.force_build = options.force_build |
| builder.force_build_failures = options.force_build_failures |
| builder.force_reconfig = options.force_reconfig |
| builder.in_tree = options.in_tree |
| |
| # Work out which boards to build |
| board_selected = boards.GetSelectedDict() |
| |
| if series: |
| commits = series.commits |
| # Number the commits for test purposes |
| for commit in range(len(commits)): |
| commits[commit].sequence = commit |
| else: |
| commits = None |
| |
| Print(GetActionSummary(options.summary, commits, board_selected, |
| options)) |
| |
| # We can't show function sizes without board details at present |
| if options.show_bloat: |
| options.show_detail = True |
| builder.SetDisplayOptions(options.show_errors, options.show_sizes, |
| options.show_detail, options.show_bloat, |
| options.list_error_boards, |
| options.show_config) |
| if options.summary: |
| builder.ShowSummary(commits, board_selected) |
| else: |
| fail, warned = builder.BuildBoards(commits, board_selected, |
| options.keep_outputs, options.verbose) |
| if fail: |
| return 128 |
| elif warned: |
| return 129 |
| return 0 |
