Skip to content
Snippets Groups Projects
builder.py 46.2 KiB
Newer Older
# Copyright (c) 2013 The Chromium OS Authors.
#
# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
#
# SPDX-License-Identifier:	GPL-2.0+
#

import collections
from datetime import datetime, timedelta
import glob
import os
import re
import Queue
import shutil
import string
import sys
import time

import builderthread
import command
import gitutil
import terminal
from terminal import Print
import toolchain


"""
Theory of Operation

Please see README for user documentation, and you should be familiar with
that before trying to make sense of this.

Buildman works by keeping the machine as busy as possible, building different
commits for different boards on multiple CPUs at once.

The source repo (self.git_dir) contains all the commits to be built. Each
thread works on a single board at a time. It checks out the first commit,
configures it for that board, then builds it. Then it checks out the next
commit and builds it (typically without re-configuring). When it runs out
of commits, it gets another job from the builder and starts again with that
board.

Clearly the builder threads could work either way - they could check out a
commit and then built it for all boards. Using separate directories for each
commit/board pair they could leave their build product around afterwards
also.

The intent behind building a single board for multiple commits, is to make
use of incremental builds. Since each commit is built incrementally from
the previous one, builds are faster. Reconfiguring for a different board
removes all intermediate object files.

Many threads can be working at once, but each has its own working directory.
When a thread finishes a build, it puts the output files into a result
directory.

The base directory used by buildman is normally '../<branch>', i.e.
a directory higher than the source repository and named after the branch
being built.

Within the base directory, we have one subdirectory for each commit. Within
that is one subdirectory for each board. Within that is the build output for
that commit/board combination.

Buildman also create working directories for each thread, in a .bm-work/
subdirectory in the base dir.

As an example, say we are building branch 'us-net' for boards 'sandbox' and
'seaboard', and say that us-net has two commits. We will have directories
like this:

us-net/             base directory
    01_of_02_g4ed4ebc_net--Add-tftp-speed-/
        sandbox/
            u-boot.bin
        seaboard/
            u-boot.bin
    02_of_02_g4ed4ebc_net--Check-tftp-comp/
        sandbox/
            u-boot.bin
        seaboard/
            u-boot.bin
    .bm-work/
        00/         working directory for thread 0 (contains source checkout)
            build/  build output
        01/         working directory for thread 1
            build/  build output
        ...
u-boot/             source directory
    .git/           repository
"""

# Possible build outcomes
OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)

# Translate a commit subject into a valid filename
trans_valid_chars = string.maketrans("/: ", "---")


class Builder:
    """Class for building U-Boot for a particular commit.

    Public members: (many should ->private)
        active: True if the builder is active and has not been stopped
        already_done: Number of builds already completed
        base_dir: Base directory to use for builder
        checkout: True to check out source, False to skip that step.
            This is used for testing.
        col: terminal.Color() object
        count: Number of commits to build
        do_make: Method to call to invoke Make
        fail: Number of builds that failed due to error
        force_build: Force building even if a build already exists
        force_config_on_failure: If a commit fails for a board, disable
            incremental building for the next commit we build for that
            board, so that we will see all warnings/errors again.
        force_build_failures: If a previously-built build (i.e. built on
            a previous run of buildman) is marked as failed, rebuild it.
        git_dir: Git directory containing source repository
        last_line_len: Length of the last line we printed (used for erasing
            it with new progress information)
        num_jobs: Number of jobs to run at once (passed to make as -j)
        num_threads: Number of builder threads to run
        out_queue: Queue of results to process
        re_make_err: Compiled regular expression for ignore_lines
        queue: Queue of jobs to run
        threads: List of active threads
        toolchains: Toolchains object to use for building
        upto: Current commit number we are building (0.count-1)
        warned: Number of builds that produced at least one warning
        force_reconfig: Reconfigure U-Boot on each comiit. This disables
            incremental building, where buildman reconfigures on the first
            commit for a baord, and then just does an incremental build for
            the following commits. In fact buildman will reconfigure and
            retry for any failing commits, so generally the only effect of
            this option is to slow things down.
        in_tree: Build U-Boot in-tree instead of specifying an output
            directory separate from the source code. This option is really
            only useful for testing in-tree builds.

    Private members:
        _base_board_dict: Last-summarised Dict of boards
        _base_err_lines: Last-summarised list of errors
        _base_warn_lines: Last-summarised list of warnings
        _build_period_us: Time taken for a single build (float object).
        _complete_delay: Expected delay until completion (timedelta)
        _next_delay_update: Next time we plan to display a progress update
                (datatime)
        _show_unknown: Show unknown boards (those not built) in summary
        _timestamps: List of timestamps for the completion of the last
            last _timestamp_count builds. Each is a datetime object.
        _timestamp_count: Number of timestamps to keep in our list.
        _working_dir: Base working directory containing all threads
    """
    class Outcome:
        """Records a build outcome for a single make invocation

        Public Members:
            rc: Outcome value (OUTCOME_...)
            err_lines: List of error lines or [] if none
            sizes: Dictionary of image size information, keyed by filename
                - Each value is itself a dictionary containing
                    values for 'text', 'data' and 'bss', being the integer
                    size in bytes of each section.
            func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
                    value is itself a dictionary:
                        key: function name
                        value: Size of function in bytes
        """
        def __init__(self, rc, err_lines, sizes, func_sizes):
            self.rc = rc
            self.err_lines = err_lines
            self.sizes = sizes
            self.func_sizes = func_sizes

    def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
                 gnu_make='make', checkout=True, show_unknown=True, step=1,
                 no_subdirs=False):
        """Create a new Builder object

        Args:
            toolchains: Toolchains object to use for building
            base_dir: Base directory to use for builder
            git_dir: Git directory containing source repository
            num_threads: Number of builder threads to run
            num_jobs: Number of jobs to run at once (passed to make as -j)
            gnu_make: the command name of GNU Make.
            checkout: True to check out source, False to skip that step.
                This is used for testing.
            show_unknown: Show unknown boards (those not built) in summary
            step: 1 to process every commit, n to process every nth commit
        """
        self.toolchains = toolchains
        self.base_dir = base_dir
        self._working_dir = os.path.join(base_dir, '.bm-work')
        self.threads = []
        self.active = True
        self.do_make = self.Make
        self.gnu_make = gnu_make
        self.checkout = checkout
Loading
Loading full blame…