Newer
Older
# Copyright (c) 2013 The Chromium OS Authors.
#
# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
#
# SPDX-License-Identifier: GPL-2.0+
#
import collections
from datetime import datetime, timedelta
import glob
import os
import re
import Queue
import shutil
import string
import sys
import time
import command
import gitutil
import terminal
from terminal import Print
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
import toolchain
"""
Theory of Operation
Please see README for user documentation, and you should be familiar with
that before trying to make sense of this.
Buildman works by keeping the machine as busy as possible, building different
commits for different boards on multiple CPUs at once.
The source repo (self.git_dir) contains all the commits to be built. Each
thread works on a single board at a time. It checks out the first commit,
configures it for that board, then builds it. Then it checks out the next
commit and builds it (typically without re-configuring). When it runs out
of commits, it gets another job from the builder and starts again with that
board.
Clearly the builder threads could work either way - they could check out a
commit and then built it for all boards. Using separate directories for each
commit/board pair they could leave their build product around afterwards
also.
The intent behind building a single board for multiple commits, is to make
use of incremental builds. Since each commit is built incrementally from
the previous one, builds are faster. Reconfiguring for a different board
removes all intermediate object files.
Many threads can be working at once, but each has its own working directory.
When a thread finishes a build, it puts the output files into a result
directory.
The base directory used by buildman is normally '../<branch>', i.e.
a directory higher than the source repository and named after the branch
being built.
Within the base directory, we have one subdirectory for each commit. Within
that is one subdirectory for each board. Within that is the build output for
that commit/board combination.
Buildman also create working directories for each thread, in a .bm-work/
subdirectory in the base dir.
As an example, say we are building branch 'us-net' for boards 'sandbox' and
'seaboard', and say that us-net has two commits. We will have directories
like this:
us-net/ base directory
01_of_02_g4ed4ebc_net--Add-tftp-speed-/
sandbox/
u-boot.bin
seaboard/
u-boot.bin
02_of_02_g4ed4ebc_net--Check-tftp-comp/
sandbox/
u-boot.bin
seaboard/
u-boot.bin
.bm-work/
00/ working directory for thread 0 (contains source checkout)
build/ build output
01/ working directory for thread 1
build/ build output
...
u-boot/ source directory
.git/ repository
"""
# Possible build outcomes
OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
# Translate a commit subject into a valid filename
trans_valid_chars = string.maketrans("/: ", "---")
class Builder:
"""Class for building U-Boot for a particular commit.
Public members: (many should ->private)
active: True if the builder is active and has not been stopped
already_done: Number of builds already completed
base_dir: Base directory to use for builder
checkout: True to check out source, False to skip that step.
This is used for testing.
col: terminal.Color() object
count: Number of commits to build
do_make: Method to call to invoke Make
fail: Number of builds that failed due to error
force_build: Force building even if a build already exists
force_config_on_failure: If a commit fails for a board, disable
incremental building for the next commit we build for that
board, so that we will see all warnings/errors again.
force_build_failures: If a previously-built build (i.e. built on
a previous run of buildman) is marked as failed, rebuild it.
git_dir: Git directory containing source repository
last_line_len: Length of the last line we printed (used for erasing
it with new progress information)
num_jobs: Number of jobs to run at once (passed to make as -j)
num_threads: Number of builder threads to run
out_queue: Queue of results to process
re_make_err: Compiled regular expression for ignore_lines
queue: Queue of jobs to run
threads: List of active threads
toolchains: Toolchains object to use for building
upto: Current commit number we are building (0.count-1)
warned: Number of builds that produced at least one warning
force_reconfig: Reconfigure U-Boot on each comiit. This disables
incremental building, where buildman reconfigures on the first
commit for a baord, and then just does an incremental build for
the following commits. In fact buildman will reconfigure and
retry for any failing commits, so generally the only effect of
this option is to slow things down.
in_tree: Build U-Boot in-tree instead of specifying an output
directory separate from the source code. This option is really
only useful for testing in-tree builds.
Private members:
_base_board_dict: Last-summarised Dict of boards
_base_err_lines: Last-summarised list of errors
_base_warn_lines: Last-summarised list of warnings
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
_build_period_us: Time taken for a single build (float object).
_complete_delay: Expected delay until completion (timedelta)
_next_delay_update: Next time we plan to display a progress update
(datatime)
_show_unknown: Show unknown boards (those not built) in summary
_timestamps: List of timestamps for the completion of the last
last _timestamp_count builds. Each is a datetime object.
_timestamp_count: Number of timestamps to keep in our list.
_working_dir: Base working directory containing all threads
"""
class Outcome:
"""Records a build outcome for a single make invocation
Public Members:
rc: Outcome value (OUTCOME_...)
err_lines: List of error lines or [] if none
sizes: Dictionary of image size information, keyed by filename
- Each value is itself a dictionary containing
values for 'text', 'data' and 'bss', being the integer
size in bytes of each section.
func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
value is itself a dictionary:
key: function name
value: Size of function in bytes
"""
def __init__(self, rc, err_lines, sizes, func_sizes):
self.rc = rc
self.err_lines = err_lines
self.sizes = sizes
self.func_sizes = func_sizes
def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
gnu_make='make', checkout=True, show_unknown=True, step=1,
no_subdirs=False):
"""Create a new Builder object
Args:
toolchains: Toolchains object to use for building
base_dir: Base directory to use for builder
git_dir: Git directory containing source repository
num_threads: Number of builder threads to run
num_jobs: Number of jobs to run at once (passed to make as -j)
gnu_make: the command name of GNU Make.
checkout: True to check out source, False to skip that step.
This is used for testing.
show_unknown: Show unknown boards (those not built) in summary
step: 1 to process every commit, n to process every nth commit
"""
self.toolchains = toolchains
self.base_dir = base_dir
self._working_dir = os.path.join(base_dir, '.bm-work')
self.threads = []
self.active = True
self.do_make = self.Make
self.checkout = checkout
Loading
Loading full blame…