moveconfig.py

        if self.ps.poll() != 0:
            self.handle_error()
        elif self.state == STATE_DEFCONFIG:
            if self.reference_src_dir and not self.current_src_dir:
                self.do_savedefconfig()
            else:
                self.do_autoconf()
        elif self.state == STATE_AUTOCONF:
            if self.current_src_dir:
                self.current_src_dir = None
                self.do_defconfig()
            else:
                self.do_savedefconfig()
        elif self.state == STATE_SAVEDEFCONFIG:
            self.update_defconfig()
        else:
            sys.exit("Internal Error. This should not happen.")

        return True if self.state == STATE_IDLE else False

    def handle_error(self):
        """Handle error cases."""

        self.log += color_text(self.options.color, COLOR_LIGHT_RED,
                               "Failed to process.\n")
        if self.options.verbose:
            self.log += color_text(self.options.color, COLOR_LIGHT_CYAN,
                                   self.ps.stderr.read())
        self.finish(False)

    def do_defconfig(self):
        """Run 'make <board>_defconfig' to create the .config file."""

        cmd = list(self.make_cmd)
        cmd.append(self.defconfig)
        self.ps = subprocess.Popen(cmd, stdout=self.devnull,
                                   stderr=subprocess.PIPE,
                                   cwd=self.current_src_dir)
        self.state = STATE_DEFCONFIG

    def do_autoconf(self):
        """Run 'make include/config/auto.conf'."""

        self.cross_compile = self.parser.get_cross_compile()
        if self.cross_compile is None:
            self.log += color_text(self.options.color, COLOR_YELLOW,
                                   "Compiler is missing.  Do nothing.\n")
            self.finish(False)
            return

        cmd = list(self.make_cmd)
        if self.cross_compile:
            cmd.append('CROSS_COMPILE=%s' % self.cross_compile)
        cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
        cmd.append('include/config/auto.conf')
        self.ps = subprocess.Popen(cmd, stdout=self.devnull,
                                   stderr=subprocess.PIPE,
                                   cwd=self.current_src_dir)
        self.state = STATE_AUTOCONF

    def do_savedefconfig(self):
        """Update the .config and run 'make savedefconfig'."""

        (updated, suspicious, log) = self.parser.update_dotconfig()
        if suspicious:
            self.suspicious_boards.add(self.defconfig)
        self.log += log

        if not self.options.force_sync and not updated:
            self.finish(True)
            return
        if updated:
            self.log += color_text(self.options.color, COLOR_LIGHT_GREEN,
                                   "Syncing by savedefconfig...\n")
        else:
            self.log += "Syncing by savedefconfig (forced by option)...\n"

        cmd = list(self.make_cmd)
        cmd.append('savedefconfig')
        self.ps = subprocess.Popen(cmd, stdout=self.devnull,
                                   stderr=subprocess.PIPE)
        self.state = STATE_SAVEDEFCONFIG

    def update_defconfig(self):
        """Update the input defconfig and go back to the idle state."""

        log = self.parser.check_defconfig()
        if log:
            self.suspicious_boards.add(self.defconfig)
            self.log += log
        orig_defconfig = os.path.join('configs', self.defconfig)
        new_defconfig = os.path.join(self.build_dir, 'defconfig')
        updated = not filecmp.cmp(orig_defconfig, new_defconfig)

        if updated:
            self.log += color_text(self.options.color, COLOR_LIGHT_BLUE,
                                   "defconfig was updated.\n")

        if not self.options.dry_run and updated:
            shutil.move(new_defconfig, orig_defconfig)
        self.finish(True)

    def finish(self, success):
        """Display log along with progress and go to the idle state.

        Arguments:
          success: Should be True when the defconfig was processed
                   successfully, or False when it fails.
        """
        # output at least 30 characters to hide the "* defconfigs out of *".
        log = self.defconfig.ljust(30) + '\n'

        log += '\n'.join([ '    ' + s for s in self.log.split('\n') ])
        # Some threads are running in parallel.
        # Print log atomically to not mix up logs from different threads.
        print >> (sys.stdout if success else sys.stderr), log

        if not success:
            if self.options.exit_on_error:
                sys.exit("Exit on error.")
            # If --exit-on-error flag is not set, skip this board and continue.
            # Record the failed board.
            self.failed_boards.add(self.defconfig)

        self.progress.inc()
        self.progress.show()
        self.state = STATE_IDLE

    def get_failed_boards(self):
        """Returns a set of failed boards (defconfigs) in this slot.
        """
        return self.failed_boards

    def get_suspicious_boards(self):
        """Returns a set of boards (defconfigs) with possible misconversion.
        """
        return self.suspicious_boards - self.failed_boards

class Slots:

    """Controller of the array of subprocess slots."""

    def __init__(self, configs, options, progress, reference_src_dir):
        """Create a new slots controller.

        Arguments:
          configs: A list of CONFIGs to move.
          options: option flags.
          progress: A progress indicator.
          reference_src_dir: Determine the true starting config state from this
                             source tree.
        """
        self.options = options
        self.slots = []
        devnull = get_devnull()
        make_cmd = get_make_cmd()
        for i in range(options.jobs):
            self.slots.append(Slot(configs, options, progress, devnull,
                                   make_cmd, reference_src_dir))

    def add(self, defconfig):
        """Add a new subprocess if a vacant slot is found.

        Arguments:
          defconfig: defconfig name to be put into.

        Returns:
          Return True on success or False on failure
        """
        for slot in self.slots:
            if slot.add(defconfig):
                return True
        return False

    def available(self):
        """Check if there is a vacant slot.

        Returns:
          Return True if at lease one vacant slot is found, False otherwise.
        """
        for slot in self.slots:
            if slot.poll():
                return True
        return False

    def empty(self):
        """Check if all slots are vacant.

        Returns:
          Return True if all the slots are vacant, False otherwise.
        """
        ret = True
        for slot in self.slots:
            if not slot.poll():
                ret = False
        return ret

    def show_failed_boards(self):
        """Display all of the failed boards (defconfigs)."""
        boards = set()
        output_file = 'moveconfig.failed'

        for slot in self.slots:
            boards |= slot.get_failed_boards()

        if boards:
            boards = '\n'.join(boards) + '\n'
            msg = "The following boards were not processed due to error:\n"
            msg += boards
            msg += "(the list has been saved in %s)\n" % output_file
            print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED,
                                            msg)

            with open(output_file, 'w') as f:
                f.write(boards)

    def show_suspicious_boards(self):
        """Display all boards (defconfigs) with possible misconversion."""
        boards = set()
        output_file = 'moveconfig.suspicious'

        for slot in self.slots:
            boards |= slot.get_suspicious_boards()

        if boards:
            boards = '\n'.join(boards) + '\n'
            msg = "The following boards might have been converted incorrectly.\n"
            msg += "It is highly recommended to check them manually:\n"
            msg += boards
            msg += "(the list has been saved in %s)\n" % output_file
            print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW,
                                            msg)

            with open(output_file, 'w') as f:
                f.write(boards)

class ReferenceSource:

    """Reference source against which original configs should be parsed."""

    def __init__(self, commit):
        """Create a reference source directory based on a specified commit.

        Arguments:
          commit: commit to git-clone
        """
        self.src_dir = tempfile.mkdtemp()
        print "Cloning git repo to a separate work directory..."
        subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
                                cwd=self.src_dir)
        print "Checkout '%s' to build the original autoconf.mk." % \
            subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip()
        subprocess.check_output(['git', 'checkout', commit],
                                stderr=subprocess.STDOUT, cwd=self.src_dir)

    def __del__(self):
        """Delete the reference source directory

        This function makes sure the temporary directory is cleaned away
        even if Python suddenly dies due to error.  It should be done in here
        because it is guaranteed the destructor is always invoked when the
        instance of the class gets unreferenced.
        """
        shutil.rmtree(self.src_dir)

    def get_dir(self):
        """Return the absolute path to the reference source directory."""

        return self.src_dir

def move_config(configs, options):
    """Move config options to defconfig files.

    Arguments:
      configs: A list of CONFIGs to move.
      options: option flags
    """
    if len(configs) == 0:
        if options.force_sync:
            print 'No CONFIG is specified. You are probably syncing defconfigs.',
        else:
            print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.',
    else:
        print 'Move ' + ', '.join(configs),
    print '(jobs: %d)\n' % options.jobs

    if options.git_ref:
        reference_src = ReferenceSource(options.git_ref)
        reference_src_dir = reference_src.get_dir()
    else:
        reference_src_dir = None

    if options.defconfigs:
        defconfigs = get_matched_defconfigs(options.defconfigs)
    else:
        defconfigs = get_all_defconfigs()

    progress = Progress(len(defconfigs))
    slots = Slots(configs, options, progress, reference_src_dir)

    # Main loop to process defconfig files:
    #  Add a new subprocess into a vacant slot.
    #  Sleep if there is no available slot.
    for defconfig in defconfigs:
        while not slots.add(defconfig):
            while not slots.available():
                # No available slot: sleep for a while
                time.sleep(SLEEP_TIME)

    # wait until all the subprocesses finish
    while not slots.empty():
        time.sleep(SLEEP_TIME)

    print ''
    slots.show_failed_boards()
    slots.show_suspicious_boards()

def main():
    try:
        cpu_count = multiprocessing.cpu_count()
    except NotImplementedError:
        cpu_count = 1

    parser = optparse.OptionParser()
    # Add options here
    parser.add_option('-c', '--color', action='store_true', default=False,
                      help='display the log in color')
    parser.add_option('-C', '--commit', action='store_true', default=False,
                      help='Create a git commit for the operation')
    parser.add_option('-d', '--defconfigs', type='string',
                      help='a file containing a list of defconfigs to move')
    parser.add_option('-n', '--dry-run', action='store_true', default=False,
                      help='perform a trial run (show log with no changes)')
    parser.add_option('-e', '--exit-on-error', action='store_true',
                      default=False,
                      help='exit immediately on any error')
    parser.add_option('-s', '--force-sync', action='store_true', default=False,
                      help='force sync by savedefconfig')
    parser.add_option('-S', '--spl', action='store_true', default=False,
                      help='parse config options defined for SPL build')
    parser.add_option('-H', '--headers-only', dest='cleanup_headers_only',
                      action='store_true', default=False,
                      help='only cleanup the headers')
    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
                      help='the number of jobs to run simultaneously')
    parser.add_option('-r', '--git-ref', type='string',
                      help='the git ref to clone for building the autoconf.mk')
    parser.add_option('-y', '--yes', action='store_true', default=False,
                      help="respond 'yes' to any prompts")
    parser.add_option('-v', '--verbose', action='store_true', default=False,
                      help='show any build errors as boards are built')
    parser.usage += ' CONFIG ...'

    (options, configs) = parser.parse_args()

    if len(configs) == 0 and not options.force_sync:
        parser.print_usage()
        sys.exit(1)

    # prefix the option name with CONFIG_ if missing
    configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config
                for config in configs ]

    check_top_directory()

    if not options.cleanup_headers_only:
        check_clean_directory()
        update_cross_compile(options.color)
        move_config(configs, options)

    if configs:
        cleanup_headers(configs, options)
        cleanup_extra_options(configs, options)
        cleanup_whitelist(configs, options)
        cleanup_readme(configs, options)

    if options.commit:
        subprocess.call(['git', 'add', '-u'])
        if configs:
            msg = 'Convert %s %sto Kconfig' % (configs[0],
                    'et al ' if len(configs) > 1 else '')
            msg += ('\n\nThis converts the following to Kconfig:\n   %s\n' %
                    '\n   '.join(configs))
        else:
            msg = 'configs: Resync with savedefconfig'
            msg += '\n\nRsync all defconfig files using moveconfig.py'
        subprocess.call(['git', 'commit', '-s', '-m', msg])

if __name__ == '__main__':
    main()