1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2013 The Chromium OS Authors.
3#
4
5import multiprocessing
6import os
7import shutil
8import subprocess
9import sys
10
11from buildman import board
12from buildman import bsettings
13from buildman import toolchain
14from buildman.builder import Builder
15from patman import command
16from patman import gitutil
17from patman import patchstream
18from patman import terminal
19from patman import tools
20from patman.terminal import Print
21
22def GetPlural(count):
23    """Returns a plural 's' if count is not 1"""
24    return 's' if count != 1 else ''
25
26def GetActionSummary(is_summary, commits, selected, options):
27    """Return a string summarising the intended action.
28
29    Returns:
30        Summary string.
31    """
32    if commits:
33        count = len(commits)
34        count = (count + options.step - 1) // options.step
35        commit_str = '%d commit%s' % (count, GetPlural(count))
36    else:
37        commit_str = 'current source'
38    str = '%s %s for %d boards' % (
39        'Summary of' if is_summary else 'Building', commit_str,
40        len(selected))
41    str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
42            GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
43    return str
44
45def ShowActions(series, why_selected, boards_selected, builder, options,
46                board_warnings):
47    """Display a list of actions that we would take, if not a dry run.
48
49    Args:
50        series: Series object
51        why_selected: Dictionary where each key is a buildman argument
52                provided by the user, and the value is the list of boards
53                brought in by that argument. For example, 'arm' might bring
54                in 400 boards, so in this case the key would be 'arm' and
55                the value would be a list of board names.
56        boards_selected: Dict of selected boards, key is target name,
57                value is Board object
58        builder: The builder that will be used to build the commits
59        options: Command line options object
60        board_warnings: List of warnings obtained from board selected
61    """
62    col = terminal.Color()
63    print('Dry run, so not doing much. But I would do this:')
64    print()
65    if series:
66        commits = series.commits
67    else:
68        commits = None
69    print(GetActionSummary(False, commits, boards_selected,
70            options))
71    print('Build directory: %s' % builder.base_dir)
72    if commits:
73        for upto in range(0, len(series.commits), options.step):
74            commit = series.commits[upto]
75            print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
76            print(commit.subject)
77    print()
78    for arg in why_selected:
79        if arg != 'all':
80            print(arg, ': %d boards' % len(why_selected[arg]))
81            if options.verbose:
82                print('   %s' % ' '.join(why_selected[arg]))
83    print(('Total boards to build for each commit: %d\n' %
84            len(why_selected['all'])))
85    if board_warnings:
86        for warning in board_warnings:
87            print(col.Color(col.YELLOW, warning))
88
89def ShowToolchainPrefix(boards, toolchains):
90    """Show information about a the tool chain used by one or more boards
91
92    The function checks that all boards use the same toolchain, then prints
93    the correct value for CROSS_COMPILE.
94
95    Args:
96        boards: Boards object containing selected boards
97        toolchains: Toolchains object containing available toolchains
98
99    Return:
100        None on success, string error message otherwise
101    """
102    boards = boards.GetSelectedDict()
103    tc_set = set()
104    for brd in boards.values():
105        tc_set.add(toolchains.Select(brd.arch))
106    if len(tc_set) != 1:
107        return 'Supplied boards must share one toolchain'
108        return False
109    tc = tc_set.pop()
110    print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
111    return None
112
113def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
114               clean_dir=False, test_thread_exceptions=False):
115    """The main control code for buildman
116
117    Args:
118        options: Command line options object
119        args: Command line arguments (list of strings)
120        toolchains: Toolchains to use - this should be a Toolchains()
121                object. If None, then it will be created and scanned
122        make_func: Make function to use for the builder. This is called
123                to execute 'make'. If this is None, the normal function
124                will be used, which calls the 'make' tool with suitable
125                arguments. This setting is useful for tests.
126        board: Boards() object to use, containing a list of available
127                boards. If this is None it will be created and scanned.
128        clean_dir: Used for tests only, indicates that the existing output_dir
129            should be removed before starting the build
130        test_thread_exceptions: Uses for tests only, True to make the threads
131            raise an exception instead of reporting their result. This simulates
132            a failure in the code somewhere
133    """
134    global builder
135
136    if options.full_help:
137        tools.PrintFullHelp(
138            os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README')
139        )
140        return 0
141
142    gitutil.Setup()
143    col = terminal.Color()
144
145    options.git_dir = os.path.join(options.git, '.git')
146
147    no_toolchains = toolchains is None
148    if no_toolchains:
149        toolchains = toolchain.Toolchains(options.override_toolchain)
150
151    if options.fetch_arch:
152        if options.fetch_arch == 'list':
153            sorted_list = toolchains.ListArchs()
154            print(col.Color(col.BLUE, 'Available architectures: %s\n' %
155                            ' '.join(sorted_list)))
156            return 0
157        else:
158            fetch_arch = options.fetch_arch
159            if fetch_arch == 'all':
160                fetch_arch = ','.join(toolchains.ListArchs())
161                print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
162                                fetch_arch))
163            for arch in fetch_arch.split(','):
164                print()
165                ret = toolchains.FetchAndInstall(arch)
166                if ret:
167                    return ret
168            return 0
169
170    if no_toolchains:
171        toolchains.GetSettings()
172        toolchains.Scan(options.list_tool_chains and options.verbose)
173    if options.list_tool_chains:
174        toolchains.List()
175        print()
176        return 0
177
178    if options.incremental:
179        print(col.Color(col.RED,
180                        'Warning: -I has been removed. See documentation'))
181    if not options.output_dir:
182        if options.work_in_output:
183            sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
184        options.output_dir = '..'
185
186    # Work out what subset of the boards we are building
187    if not boards:
188        if not os.path.exists(options.output_dir):
189            os.makedirs(options.output_dir)
190        board_file = os.path.join(options.output_dir, 'boards.cfg')
191        our_path = os.path.dirname(os.path.realpath(__file__))
192        genboardscfg = os.path.join(our_path, '../genboardscfg.py')
193        if not os.path.exists(genboardscfg):
194            genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
195        status = subprocess.call([genboardscfg, '-q', '-o', board_file])
196        if status != 0:
197            # Older versions don't support -q
198            status = subprocess.call([genboardscfg, '-o', board_file])
199            if status != 0:
200                sys.exit("Failed to generate boards.cfg")
201
202        boards = board.Boards()
203        boards.ReadBoards(board_file)
204
205    exclude = []
206    if options.exclude:
207        for arg in options.exclude:
208            exclude += arg.split(',')
209
210    if options.boards:
211        requested_boards = []
212        for b in options.boards:
213            requested_boards += b.split(',')
214    else:
215        requested_boards = None
216    why_selected, board_warnings = boards.SelectBoards(args, exclude,
217                                                       requested_boards)
218    selected = boards.GetSelected()
219    if not len(selected):
220        sys.exit(col.Color(col.RED, 'No matching boards found'))
221
222    if options.print_prefix:
223        err = ShowToolchainPrefix(boards, toolchains)
224        if err:
225            sys.exit(col.Color(col.RED, err))
226        return 0
227
228    # Work out how many commits to build. We want to build everything on the
229    # branch. We also build the upstream commit as a control so we can see
230    # problems introduced by the first commit on the branch.
231    count = options.count
232    has_range = options.branch and '..' in options.branch
233    if count == -1:
234        if not options.branch:
235            count = 1
236        else:
237            if has_range:
238                count, msg = gitutil.CountCommitsInRange(options.git_dir,
239                                                         options.branch)
240            else:
241                count, msg = gitutil.CountCommitsInBranch(options.git_dir,
242                                                          options.branch)
243            if count is None:
244                sys.exit(col.Color(col.RED, msg))
245            elif count == 0:
246                sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
247                                   options.branch))
248            if msg:
249                print(col.Color(col.YELLOW, msg))
250            count += 1   # Build upstream commit also
251
252    if not count:
253        str = ("No commits found to process in branch '%s': "
254               "set branch's upstream or use -c flag" % options.branch)
255        sys.exit(col.Color(col.RED, str))
256    if options.work_in_output:
257        if len(selected) != 1:
258            sys.exit(col.Color(col.RED,
259                               '-w can only be used with a single board'))
260        if count != 1:
261            sys.exit(col.Color(col.RED,
262                               '-w can only be used with a single commit'))
263
264    # Read the metadata from the commits. First look at the upstream commit,
265    # then the ones in the branch. We would like to do something like
266    # upstream/master~..branch but that isn't possible if upstream/master is
267    # a merge commit (it will list all the commits that form part of the
268    # merge)
269    # Conflicting tags are not a problem for buildman, since it does not use
270    # them. For example, Series-version is not useful for buildman. On the
271    # other hand conflicting tags will cause an error. So allow later tags
272    # to overwrite earlier ones by setting allow_overwrite=True
273    if options.branch:
274        if count == -1:
275            if has_range:
276                range_expr = options.branch
277            else:
278                range_expr = gitutil.GetRangeInBranch(options.git_dir,
279                                                      options.branch)
280            upstream_commit = gitutil.GetUpstream(options.git_dir,
281                                                  options.branch)
282            series = patchstream.get_metadata_for_list(upstream_commit,
283                options.git_dir, 1, series=None, allow_overwrite=True)
284
285            series = patchstream.get_metadata_for_list(range_expr,
286                    options.git_dir, None, series, allow_overwrite=True)
287        else:
288            # Honour the count
289            series = patchstream.get_metadata_for_list(options.branch,
290                    options.git_dir, count, series=None, allow_overwrite=True)
291    else:
292        series = None
293        if not options.dry_run:
294            options.verbose = True
295            if not options.summary:
296                options.show_errors = True
297
298    # By default we have one thread per CPU. But if there are not enough jobs
299    # we can have fewer threads and use a high '-j' value for make.
300    if options.threads is None:
301        options.threads = min(multiprocessing.cpu_count(), len(selected))
302    if not options.jobs:
303        options.jobs = max(1, (multiprocessing.cpu_count() +
304                len(selected) - 1) // len(selected))
305
306    if not options.step:
307        options.step = len(series.commits) - 1
308
309    gnu_make = command.Output(os.path.join(options.git,
310            'scripts/show-gnu-make'), raise_on_error=False).rstrip()
311    if not gnu_make:
312        sys.exit('GNU Make not found')
313
314    # Create a new builder with the selected options.
315    output_dir = options.output_dir
316    if options.branch:
317        dirname = options.branch.replace('/', '_')
318        # As a special case allow the board directory to be placed in the
319        # output directory itself rather than any subdirectory.
320        if not options.no_subdirs:
321            output_dir = os.path.join(options.output_dir, dirname)
322        if clean_dir and os.path.exists(output_dir):
323            shutil.rmtree(output_dir)
324    builder = Builder(toolchains, output_dir, options.git_dir,
325            options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
326            show_unknown=options.show_unknown, step=options.step,
327            no_subdirs=options.no_subdirs, full_path=options.full_path,
328            verbose_build=options.verbose_build,
329            mrproper=options.mrproper,
330            per_board_out_dir=options.per_board_out_dir,
331            config_only=options.config_only,
332            squash_config_y=not options.preserve_config_y,
333            warnings_as_errors=options.warnings_as_errors,
334            work_in_output=options.work_in_output,
335            test_thread_exceptions=test_thread_exceptions)
336    builder.force_config_on_failure = not options.quick
337    if make_func:
338        builder.do_make = make_func
339
340    # For a dry run, just show our actions as a sanity check
341    if options.dry_run:
342        ShowActions(series, why_selected, selected, builder, options,
343                    board_warnings)
344    else:
345        builder.force_build = options.force_build
346        builder.force_build_failures = options.force_build_failures
347        builder.force_reconfig = options.force_reconfig
348        builder.in_tree = options.in_tree
349
350        # Work out which boards to build
351        board_selected = boards.GetSelectedDict()
352
353        if series:
354            commits = series.commits
355            # Number the commits for test purposes
356            for commit in range(len(commits)):
357                commits[commit].sequence = commit
358        else:
359            commits = None
360
361        Print(GetActionSummary(options.summary, commits, board_selected,
362                               options))
363
364        # We can't show function sizes without board details at present
365        if options.show_bloat:
366            options.show_detail = True
367        builder.SetDisplayOptions(
368            options.show_errors, options.show_sizes, options.show_detail,
369            options.show_bloat, options.list_error_boards, options.show_config,
370            options.show_environment, options.filter_dtb_warnings,
371            options.filter_migration_warnings)
372        if options.summary:
373            builder.ShowSummary(commits, board_selected)
374        else:
375            fail, warned, excs = builder.BuildBoards(
376                commits, board_selected, options.keep_outputs, options.verbose)
377            if excs:
378                return 102
379            elif fail:
380                return 100
381            elif warned and not options.ignore_warnings:
382                return 101
383    return 0
384