builder.py

        """Reset the results summary ready for use.

        Set up the base board list to be all those selected, and set the
        error lines to empty.

        Following this, calls to PrintResultSummary() will use this
        information to work out what has changed.

        Args:
            board_selected: Dict containing boards to summarise, keyed by
                board.target
        """
        self._base_board_dict = {}
        for board in board_selected:
            self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
        self._base_err_lines = []

    def PrintFuncSizeDetail(self, fname, old, new):
        grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
        delta, common = [], {}

        for a in old:
            if a in new:
                common[a] = 1

        for name in old:
            if name not in common:
                remove += 1
                down += old[name]
                delta.append([-old[name], name])

        for name in new:
            if name not in common:
                add += 1
                up += new[name]
                delta.append([new[name], name])

        for name in common:
                diff = new.get(name, 0) - old.get(name, 0)
                if diff > 0:
                    grow, up = grow + 1, up + diff
                elif diff < 0:
                    shrink, down = shrink + 1, down - diff
                delta.append([diff, name])

        delta.sort()
        delta.reverse()

        args = [add, -remove, grow, -shrink, up, -down, up - down]
        if max(args) == 0:
            return
        args = [self.ColourNum(x) for x in args]
        indent = ' ' * 15
        print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
        print '%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
                                        'delta')
        for diff, name in delta:
            if diff:
                color = self.col.RED if diff > 0 else self.col.GREEN
                msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
                        old.get(name, '-'), new.get(name,'-'), diff)
                print self.col.Color(color, msg)


    def PrintSizeDetail(self, target_list, show_bloat):
        """Show details size information for each board

        Args:
            target_list: List of targets, each a dict containing:
                    'target': Target name
                    'total_diff': Total difference in bytes across all areas
                    <part_name>: Difference for that part
            show_bloat: Show detail for each function
        """
        targets_by_diff = sorted(target_list, reverse=True,
        key=lambda x: x['_total_diff'])
        for result in targets_by_diff:
            printed_target = False
            for name in sorted(result):
                diff = result[name]
                if name.startswith('_'):
                    continue
                if diff != 0:
                    color = self.col.RED if diff > 0 else self.col.GREEN
                msg = ' %s %+d' % (name, diff)
                if not printed_target:
                    print '%10s  %-15s:' % ('', result['_target']),
                    printed_target = True
                print self.col.Color(color, msg),
            if printed_target:
                print
                if show_bloat:
                    target = result['_target']
                    outcome = result['_outcome']
                    base_outcome = self._base_board_dict[target]
                    for fname in outcome.func_sizes:
                        self.PrintFuncSizeDetail(fname,
                                                 base_outcome.func_sizes[fname],
                                                 outcome.func_sizes[fname])


    def PrintSizeSummary(self, board_selected, board_dict, show_detail,
                         show_bloat):
        """Print a summary of image sizes broken down by section.

        The summary takes the form of one line per architecture. The
        line contains deltas for each of the sections (+ means the section
        got bigger, - means smaller). The nunmbers are the average number
        of bytes that a board in this section increased by.

        For example:
           powerpc: (622 boards)   text -0.0
          arm: (285 boards)   text -0.0
          nds32: (3 boards)   text -8.0

        Args:
            board_selected: Dict containing boards to summarise, keyed by
                board.target
            board_dict: Dict containing boards for which we built this
                commit, keyed by board.target. The value is an Outcome object.
            show_detail: Show detail for each board
            show_bloat: Show detail for each function
        """
        arch_list = {}
        arch_count = {}

        # Calculate changes in size for different image parts
        # The previous sizes are in Board.sizes, for each board
        for target in board_dict:
            if target not in board_selected:
                continue
            base_sizes = self._base_board_dict[target].sizes
            outcome = board_dict[target]
            sizes = outcome.sizes

            # Loop through the list of images, creating a dict of size
            # changes for each image/part. We end up with something like
            # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
            # which means that U-Boot data increased by 5 bytes and SPL
            # text decreased by 4.
            err = {'_target' : target}
            for image in sizes:
                if image in base_sizes:
                    base_image = base_sizes[image]
                    # Loop through the text, data, bss parts
                    for part in sorted(sizes[image]):
                        diff = sizes[image][part] - base_image[part]
                        col = None
                        if diff:
                            if image == 'u-boot':
                                name = part
                            else:
                                name = image + ':' + part
                            err[name] = diff
            arch = board_selected[target].arch
            if not arch in arch_count:
                arch_count[arch] = 1
            else:
                arch_count[arch] += 1
            if not sizes:
                pass    # Only add to our list when we have some stats
            elif not arch in arch_list:
                arch_list[arch] = [err]
            else:
                arch_list[arch].append(err)

        # We now have a list of image size changes sorted by arch
        # Print out a summary of these
        for arch, target_list in arch_list.iteritems():
            # Get total difference for each type
            totals = {}
            for result in target_list:
                total = 0
                for name, diff in result.iteritems():
                    if name.startswith('_'):
                        continue
                    total += diff
                    if name in totals:
                        totals[name] += diff
                    else:
                        totals[name] = diff
                result['_total_diff'] = total
                result['_outcome'] = board_dict[result['_target']]

            count = len(target_list)
            printed_arch = False
            for name in sorted(totals):
                diff = totals[name]
                if diff:
                    # Display the average difference in this name for this
                    # architecture
                    avg_diff = float(diff) / count
                    color = self.col.RED if avg_diff > 0 else self.col.GREEN
                    msg = ' %s %+1.1f' % (name, avg_diff)
                    if not printed_arch:
                        print '%10s: (for %d/%d boards)' % (arch, count,
                                arch_count[arch]),
                        printed_arch = True
                    print self.col.Color(color, msg),

            if printed_arch:
                print
                if show_detail:
                    self.PrintSizeDetail(target_list, show_bloat)


    def PrintResultSummary(self, board_selected, board_dict, err_lines,
                           show_sizes, show_detail, show_bloat):
        """Compare results with the base results and display delta.

        Only boards mentioned in board_selected will be considered. This
        function is intended to be called repeatedly with the results of
        each commit. It therefore shows a 'diff' between what it saw in
        the last call and what it sees now.

        Args:
            board_selected: Dict containing boards to summarise, keyed by
                board.target
            board_dict: Dict containing boards for which we built this
                commit, keyed by board.target. The value is an Outcome object.
            err_lines: A list of errors for this commit, or [] if there is
                none, or we don't want to print errors
            show_sizes: Show image size deltas
            show_detail: Show detail for each board
            show_bloat: Show detail for each function
        """
        better = []     # List of boards fixed since last commit
        worse = []      # List of new broken boards since last commit
        new = []        # List of boards that didn't exist last time
        unknown = []    # List of boards that were not built

        for target in board_dict:
            if target not in board_selected:
                continue

            # If the board was built last time, add its outcome to a list
            if target in self._base_board_dict:
                base_outcome = self._base_board_dict[target].rc
                outcome = board_dict[target]
                if outcome.rc == OUTCOME_UNKNOWN:
                    unknown.append(target)
                elif outcome.rc < base_outcome:
                    better.append(target)
                elif outcome.rc > base_outcome:
                    worse.append(target)
            else:
                new.append(target)

        # Get a list of errors that have appeared, and disappeared
        better_err = []
        worse_err = []
        for line in err_lines:
            if line not in self._base_err_lines:
                worse_err.append('+' + line)
        for line in self._base_err_lines:
            if line not in err_lines:
                better_err.append('-' + line)

        # Display results by arch
        if better or worse or unknown or new or worse_err or better_err:
            arch_list = {}
            self.AddOutcome(board_selected, arch_list, better, '',
                    self.col.GREEN)
            self.AddOutcome(board_selected, arch_list, worse, '+',
                    self.col.RED)
            self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
            if self._show_unknown:
                self.AddOutcome(board_selected, arch_list, unknown, '?',
                        self.col.MAGENTA)
            for arch, target_list in arch_list.iteritems():
                print '%10s: %s' % (arch, target_list)
            if better_err:
                print self.col.Color(self.col.GREEN, '\n'.join(better_err))
            if worse_err:
                print self.col.Color(self.col.RED, '\n'.join(worse_err))

        if show_sizes:
            self.PrintSizeSummary(board_selected, board_dict, show_detail,
                                  show_bloat)

        # Save our updated information for the next call to this function
        self._base_board_dict = board_dict
        self._base_err_lines = err_lines

        # Get a list of boards that did not get built, if needed
        not_built = []
        for board in board_selected:
            if not board in board_dict:
                not_built.append(board)
        if not_built:
            print "Boards not built (%d): %s" % (len(not_built),
                    ', '.join(not_built))


    def ShowSummary(self, commits, board_selected, show_errors, show_sizes,
                    show_detail, show_bloat):
        """Show a build summary for U-Boot for a given board list.

        Reset the result summary, then repeatedly call GetResultSummary on
        each commit's results, then display the differences we see.

        Args:
            commit: Commit objects to summarise
            board_selected: Dict containing boards to summarise
            show_errors: Show errors that occured
            show_sizes: Show size deltas
            show_detail: Show detail for each board
            show_bloat: Show detail for each function
        """
        self.commit_count = len(commits)
        self.commits = commits
        self.ResetResultSummary(board_selected)

        for commit_upto in range(0, self.commit_count, self._step):
            board_dict, err_lines = self.GetResultSummary(board_selected,
                    commit_upto, read_func_sizes=show_bloat)
            msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject)
            print self.col.Color(self.col.BLUE, msg)
            self.PrintResultSummary(board_selected, board_dict,
                    err_lines if show_errors else [], show_sizes, show_detail,
                    show_bloat)


    def SetupBuild(self, board_selected, commits):
        """Set up ready to start a build.

        Args:
            board_selected: Selected boards to build
            commits: Selected commits to build
        """
        # First work out how many commits we will build
        count = (len(commits) + self._step - 1) / self._step
        self.count = len(board_selected) * count
        self.upto = self.warned = self.fail = 0
        self._timestamps = collections.deque()

    def BuildBoardsForCommit(self, board_selected, keep_outputs):
        """Build all boards for a single commit"""
        self.SetupBuild(board_selected)
        self.count = len(board_selected)
        for brd in board_selected.itervalues():
            job = BuilderJob()
            job.board = brd
            job.commits = None
            job.keep_outputs = keep_outputs
            self.queue.put(brd)

        self.queue.join()
        self.out_queue.join()
        print
        self.ClearLine(0)

    def BuildCommits(self, commits, board_selected, show_errors, keep_outputs):
        """Build all boards for all commits (non-incremental)"""
        self.commit_count = len(commits)

        self.ResetResultSummary(board_selected)
        for self.commit_upto in range(self.commit_count):
            self.SelectCommit(commits[self.commit_upto])
            self.SelectOutputDir()
            Mkdir(self.output_dir)

            self.BuildBoardsForCommit(board_selected, keep_outputs)
            board_dict, err_lines = self.GetResultSummary()
            self.PrintResultSummary(board_selected, board_dict,
                err_lines if show_errors else [])

        if self.already_done:
            print '%d builds already done' % self.already_done

    def GetThreadDir(self, thread_num):
        """Get the directory path to the working dir for a thread.

        Args:
            thread_num: Number of thread to check.
        """
        return os.path.join(self._working_dir, '%02d' % thread_num)

    def _PrepareThread(self, thread_num):
        """Prepare the working directory for a thread.

        This clones or fetches the repo into the thread's work directory.

        Args:
            thread_num: Thread number (0, 1, ...)
        """
        thread_dir = self.GetThreadDir(thread_num)
        Mkdir(thread_dir)
        git_dir = os.path.join(thread_dir, '.git')

        # Clone the repo if it doesn't already exist
        # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
        # we have a private index but uses the origin repo's contents?
        if self.git_dir:
            src_dir = os.path.abspath(self.git_dir)
            if os.path.exists(git_dir):
                gitutil.Fetch(git_dir, thread_dir)
            else:
                print 'Cloning repo for thread %d' % thread_num
                gitutil.Clone(src_dir, thread_dir)

    def _PrepareWorkingSpace(self, max_threads):
        """Prepare the working directory for use.

        Set up the git repo for each thread.

        Args:
            max_threads: Maximum number of threads we expect to need.
        """
        Mkdir(self._working_dir)
        for thread in range(max_threads):
            self._PrepareThread(thread)

    def _PrepareOutputSpace(self):
        """Get the output directories ready to receive files.

        We delete any output directories which look like ones we need to
        create. Having left over directories is confusing when the user wants
        to check the output manually.
        """
        dir_list = []
        for commit_upto in range(self.commit_count):
            dir_list.append(self._GetOutputDir(commit_upto))

        for dirname in glob.glob(os.path.join(self.base_dir, '*')):
            if dirname not in dir_list:
                shutil.rmtree(dirname)

    def BuildBoards(self, commits, board_selected, show_errors, keep_outputs):
        """Build all commits for a list of boards

        Args:
            commits: List of commits to be build, each a Commit object
            boards_selected: Dict of selected boards, key is target name,
                    value is Board object
            show_errors: True to show summarised error/warning info
            keep_outputs: True to save build output files
        """
        self.commit_count = len(commits)
        self.commits = commits

        self.ResetResultSummary(board_selected)
        Mkdir(self.base_dir)
        self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)))
        self._PrepareOutputSpace()
        self.SetupBuild(board_selected, commits)
        self.ProcessResult(None)

        # Create jobs to build all commits for each board
        for brd in board_selected.itervalues():
            job = BuilderJob()
            job.board = brd
            job.commits = commits
            job.keep_outputs = keep_outputs
            job.step = self._step
            self.queue.put(job)

        # Wait until all jobs are started
        self.queue.join()

        # Wait until we have processed all output
        self.out_queue.join()
        print
        self.ClearLine(0)