genboardscfg.py

#!/usr/bin/env python
#
# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
#
# SPDX-License-Identifier:	GPL-2.0+
#

"""
Converter from Kconfig and MAINTAINERS to boards.cfg

Run 'tools/genboardscfg.py' to create boards.cfg file.

Run 'tools/genboardscfg.py -h' for available options.
"""

import errno
import fnmatch
import glob
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time

BOARD_FILE = 'boards.cfg'
CONFIG_DIR = 'configs'
REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
                '-i', '-d', '-', '-s', '8']
SHOW_GNU_MAKE = 'scripts/show-gnu-make'
SLEEP_TIME=0.003

COMMENT_BLOCK = '''#
# List of boards
#   Automatically generated by %s: don't edit
#
# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers

''' % __file__

### helper functions ###
def get_terminal_columns():
    """Get the width of the terminal.

    Returns:
      The width of the terminal, or zero if the stdout is not
      associated with tty.
    """
    try:
        return shutil.get_terminal_size().columns # Python 3.3~
    except AttributeError:
        import fcntl
        import termios
        import struct
        arg = struct.pack('hhhh', 0, 0, 0, 0)
        try:
            ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
        except IOError as exception:
            # If 'Inappropriate ioctl for device' error occurs,
            # stdout is probably redirected. Return 0.
            return 0
        return struct.unpack('hhhh', ret)[1]

def get_devnull():
    """Get the file object of '/dev/null' device."""
    try:
        devnull = subprocess.DEVNULL # py3k
    except AttributeError:
        devnull = open(os.devnull, 'wb')
    return devnull

def check_top_directory():
    """Exit if we are not at the top of source directory."""
    for f in ('README', 'Licenses'):
        if not os.path.exists(f):
            sys.exit('Please run at the top of source directory.')

def get_make_cmd():
    """Get the command name of GNU Make."""
    process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
    ret = process.communicate()
    if process.returncode:
        sys.exit('GNU Make not found')
    return ret[0].rstrip()

def output_is_new():
    """Check if the boards.cfg file is up to date.

    Returns:
      True if the boards.cfg file exists and is newer than any of
      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
    """
    try:
        ctime = os.path.getctime(BOARD_FILE)
    except OSError as exception:
        if exception.errno == errno.ENOENT:
            # return False on 'No such file or directory' error
            return False
        else:
            raise

    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
        for filename in fnmatch.filter(filenames, '*_defconfig'):
            if fnmatch.fnmatch(filename, '.*'):
                continue
            filepath = os.path.join(dirpath, filename)
            if ctime < os.path.getctime(filepath):
                return False

    for (dirpath, dirnames, filenames) in os.walk('.'):
        for filename in filenames:
            if (fnmatch.fnmatch(filename, '*~') or
                not fnmatch.fnmatch(filename, 'Kconfig*') and
                not filename == 'MAINTAINERS'):
                continue
            filepath = os.path.join(dirpath, filename)
            if ctime < os.path.getctime(filepath):
                return False

    # Detect a board that has been removed since the current boards.cfg
    # was generated
    with open(BOARD_FILE) as f:
        for line in f:
            if line[0] == '#' or line == '\n':
                continue
            defconfig = line.split()[6] + '_defconfig'
            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
                return False

    return True

### classes ###
class MaintainersDatabase:

    """The database of board status and maintainers."""

    def __init__(self):
        """Create an empty database."""
        self.database = {}

    def get_status(self, target):
        """Return the status of the given board.

        Returns:
          Either 'Active' or 'Orphan'
        """
        if not target in self.database:
            print >> sys.stderr, "WARNING: no status info for '%s'" % target
            return '-'

        tmp = self.database[target][0]
        if tmp.startswith('Maintained'):
            return 'Active'
        elif tmp.startswith('Orphan'):
            return 'Orphan'
        else:
            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
                                  (tmp, target))
            return '-'

    def get_maintainers(self, target):
        """Return the maintainers of the given board.

        If the board has two or more maintainers, they are separated
        with colons.
        """
        if not target in self.database:
            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
            return ''

        return ':'.join(self.database[target][1])

    def parse_file(self, file):
        """Parse the given MAINTAINERS file.

        This method parses MAINTAINERS and add board status and
        maintainers information to the database.

        Arguments:
          file: MAINTAINERS file to be parsed
        """
        targets = []
        maintainers = []
        status = '-'
        for line in open(file):
            tag, rest = line[:2], line[2:].strip()
            if tag == 'M:':
                maintainers.append(rest)
            elif tag == 'F:':
                # expand wildcard and filter by 'configs/*_defconfig'
                for f in glob.glob(rest):
                    front, match, rear = f.partition('configs/')
                    if not front and match:
                        front, match, rear = rear.rpartition('_defconfig')
                        if match and not rear:
                            targets.append(front)
            elif tag == 'S:':
                status = rest
            elif line == '\n':
                for target in targets:
                    self.database[target] = (status, maintainers)
                targets = []
                maintainers = []
                status = '-'
        if targets:
            for target in targets:
                self.database[target] = (status, maintainers)

class DotConfigParser:

    """A parser of .config file.

    Each line of the output should have the form of:
    Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
    Most of them are extracted from .config file.
    MAINTAINERS files are also consulted for Status and Maintainers fields.
    """

    re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
    re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
    re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
    re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
    re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
    re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
    re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
    re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
               ('vendor', re_vendor), ('board', re_board),
               ('config', re_config), ('options', re_options))
    must_fields = ('arch', 'config')

    def __init__(self, build_dir, output, maintainers_database):
        """Create a new .config perser.

        Arguments:
          build_dir: Build directory where .config is located
          output: File object which the result is written to
          maintainers_database: An instance of class MaintainersDatabase
        """
        self.dotconfig = os.path.join(build_dir, '.config')
        self.output = output
        self.database = maintainers_database

    def parse(self, defconfig):
        """Parse .config file and output one-line database for the given board.

        Arguments:
          defconfig: Board (defconfig) name
        """
        fields = {}
        for line in open(self.dotconfig):
            if not line.startswith('CONFIG_SYS_'):
                continue
            for (key, pattern) in self.re_list:
                m = pattern.match(line)
                if m and m.group(1):
                    fields[key] = m.group(1)
                    break

        # sanity check of '.config' file
        for field in self.must_fields:
            if not field in fields:
                print >> sys.stderr, (
                    "WARNING: '%s' is not defined in '%s'. Skip." %
                    (field, defconfig))
                return

        # fix-up for aarch64
        if fields['arch'] == 'arm' and 'cpu' in fields:
            if fields['cpu'] == 'armv8':
                fields['arch'] = 'aarch64'

        target, match, rear = defconfig.partition('_defconfig')
        assert match and not rear, \
                                '%s : invalid defconfig file name' % defconfig

        fields['status'] = self.database.get_status(target)
        fields['maintainers'] = self.database.get_maintainers(target)

        if 'options' in fields:
            options = fields['config'] + ':' + \
                      fields['options'].replace(r'\"', '"')
        elif fields['config'] != target:
            options = fields['config']
        else:
            options = '-'

        self.output.write((' '.join(['%s'] * 9) + '\n')  %
                          (fields['status'],
                           fields['arch'],
                           fields.get('cpu', '-'),
                           fields.get('soc', '-'),
                           fields.get('vendor', '-'),
                           fields.get('board', '-'),
                           target,
                           options,
                           fields['maintainers']))

class Slot:

    """A slot to store a subprocess.

    Each instance of this class handles one subprocess.
    This class is useful to control multiple processes
    for faster processing.
    """

    def __init__(self, output, maintainers_database, devnull, make_cmd):
        """Create a new slot.

        Arguments:
          output: File object which the result is written to
          maintainers_database: An instance of class MaintainersDatabase
          devnull: file object of 'dev/null'
          make_cmd: the command name of Make
        """
        self.build_dir = tempfile.mkdtemp()
        self.devnull = devnull
        self.ps = subprocess.Popen([make_cmd, 'O=' + self.build_dir,
                                    'allnoconfig'], stdout=devnull)
        self.occupied = True
        self.parser = DotConfigParser(self.build_dir, output,
                                      maintainers_database)
        self.env = os.environ.copy()
        self.env['srctree'] = os.getcwd()
        self.env['UBOOTVERSION'] = 'dummy'
        self.env['KCONFIG_OBJDIR'] = ''

    def __del__(self):
        """Delete the working directory"""
        if not self.occupied:
            while self.ps.poll() == None:
                pass
        shutil.rmtree(self.build_dir)

    def add(self, defconfig):
        """Add a new subprocess to the slot.

        Fails if the slot is occupied, that is, the current subprocess
        is still running.

        Arguments:
          defconfig: Board (defconfig) name

        Returns:
          Return True on success or False on fail
        """
        if self.occupied:
            return False

        with open(os.path.join(self.build_dir, '.tmp_defconfig'), 'w') as f:
            for line in open(os.path.join(CONFIG_DIR, defconfig)):
                colon = line.find(':CONFIG_')
                if colon == -1:
                    f.write(line)
                else:
                    f.write(line[colon + 1:])

        self.ps = subprocess.Popen([os.path.join('scripts', 'kconfig', 'conf'),
                                    '--defconfig=.tmp_defconfig', 'Kconfig'],
                                   stdout=self.devnull,
                                   cwd=self.build_dir,
                                   env=self.env)

        self.defconfig = defconfig
        self.occupied = True
        return True

    def wait(self):
        """Wait until the current subprocess finishes."""
        while self.occupied and self.ps.poll() == None:
            time.sleep(SLEEP_TIME)
        self.occupied = False

    def poll(self):
        """Check if the subprocess is running and invoke the .config
        parser if the subprocess is terminated.

        Returns:
          Return True if the subprocess is terminated, False otherwise
        """
        if not self.occupied:
            return True
        if self.ps.poll() == None:
            return False
        if self.ps.poll() == 0:
            self.parser.parse(self.defconfig)
        else:
            print >> sys.stderr, ("WARNING: failed to process '%s'. skip." %
                                  self.defconfig)
        self.occupied = False
        return True

class Slots:

    """Controller of the array of subprocess slots."""

    def __init__(self, jobs, output, maintainers_database):
        """Create a new slots controller.

        Arguments:
          jobs: A number of slots to instantiate
          output: File object which the result is written to
          maintainers_database: An instance of class MaintainersDatabase
        """
        self.slots = []
        devnull = get_devnull()
        make_cmd = get_make_cmd()
        for i in range(jobs):
            self.slots.append(Slot(output, maintainers_database,
                                   devnull, make_cmd))
        for slot in self.slots:
            slot.wait()

    def add(self, defconfig):
        """Add a new subprocess if a vacant slot is available.

        Arguments:
          defconfig: Board (defconfig) name

        Returns:
          Return True on success or False on fail
        """
        for slot in self.slots:
            if slot.add(defconfig):
                return True
        return False

    def available(self):
        """Check if there is a vacant slot.

        Returns:
          Return True if a vacant slot is found, False if all slots are full
        """
        for slot in self.slots:
            if slot.poll():
                return True
        return False

    def empty(self):
        """Check if all slots are vacant.

        Returns:
          Return True if all slots are vacant, False if at least one slot
          is running
        """
        ret = True
        for slot in self.slots:
            if not slot.poll():
                ret = False
        return ret

class Indicator:

    """A class to control the progress indicator."""

    MIN_WIDTH = 15
    MAX_WIDTH = 70

    def __init__(self, total):
        """Create an instance.

        Arguments:
          total: A number of boards
        """
        self.total = total
        self.cur = 0
        width = get_terminal_columns()
        width = min(width, self.MAX_WIDTH)
        width -= self.MIN_WIDTH
        if width > 0:
            self.enabled = True
        else:
            self.enabled = False
        self.width = width

    def inc(self):
        """Increment the counter and show the progress bar."""
        if not self.enabled:
            return
        self.cur += 1
        arrow_len = self.width * self.cur // self.total
        msg = '%4d/%d [' % (self.cur, self.total)
        msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
        sys.stdout.write('\r' + msg)
        sys.stdout.flush()

class BoardsFileGenerator:

    """Generator of boards.cfg."""

    def __init__(self):
        """Prepare basic things for generating boards.cfg."""
        # All the defconfig files to be processed
        defconfigs = []
        for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
            dirpath = dirpath[len(CONFIG_DIR) + 1:]
            for filename in fnmatch.filter(filenames, '*_defconfig'):
                if fnmatch.fnmatch(filename, '.*'):
                    continue
                defconfigs.append(os.path.join(dirpath, filename))
        self.defconfigs = defconfigs
        self.indicator = Indicator(len(defconfigs))

        # Parse all the MAINTAINERS files
        maintainers_database = MaintainersDatabase()
        for (dirpath, dirnames, filenames) in os.walk('.'):
            if 'MAINTAINERS' in filenames:
                maintainers_database.parse_file(os.path.join(dirpath,
                                                             'MAINTAINERS'))
        self.maintainers_database = maintainers_database

    def __del__(self):
        """Delete the incomplete boards.cfg

        This destructor deletes boards.cfg if the private member 'in_progress'
        is defined as True.  The 'in_progress' member is set to True at the
        beginning of the generate() method and set to False at its end.
        So, in_progress==True means generating boards.cfg was terminated
        on the way.
        """

        if hasattr(self, 'in_progress') and self.in_progress:
            try:
                os.remove(BOARD_FILE)
            except OSError as exception:
                # Ignore 'No such file or directory' error
                if exception.errno != errno.ENOENT:
                    raise
            print 'Removed incomplete %s' % BOARD_FILE

    def generate(self, jobs):
        """Generate boards.cfg

        This method sets the 'in_progress' member to True at the beginning
        and sets it to False on success.  The boards.cfg should not be
        touched before/after this method because 'in_progress' is used
        to detect the incomplete boards.cfg.

        Arguments:
          jobs: The number of jobs to run simultaneously
        """

        self.in_progress = True
        print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)

        # Output lines should be piped into the reformat tool
        reformat_process = subprocess.Popen(REFORMAT_CMD,
                                            stdin=subprocess.PIPE,
                                            stdout=open(BOARD_FILE, 'w'))
        pipe = reformat_process.stdin
        pipe.write(COMMENT_BLOCK)

        slots = Slots(jobs, pipe, self.maintainers_database)

        # Main loop to process defconfig files:
        #  Add a new subprocess into a vacant slot.
        #  Sleep if there is no available slot.
        for defconfig in self.defconfigs:
            while not slots.add(defconfig):
                while not slots.available():
                    # No available slot: sleep for a while
                    time.sleep(SLEEP_TIME)
            self.indicator.inc()

        # wait until all the subprocesses finish
        while not slots.empty():
            time.sleep(SLEEP_TIME)
        print ''

        # wait until the reformat tool finishes
        reformat_process.communicate()
        if reformat_process.returncode != 0:
            sys.exit('"%s" failed' % REFORMAT_CMD[0])

        self.in_progress = False

def gen_boards_cfg(jobs=1, force=False):
    """Generate boards.cfg file.

    The incomplete boards.cfg is deleted if an error (including
    the termination by the keyboard interrupt) occurs on the halfway.

    Arguments:
      jobs: The number of jobs to run simultaneously
    """
    check_top_directory()
    if not force and output_is_new():
        print "%s is up to date. Nothing to do." % BOARD_FILE
        sys.exit(0)

    generator = BoardsFileGenerator()
    generator.generate(jobs)

def main():
    parser = optparse.OptionParser()
    # Add options here
    parser.add_option('-j', '--jobs',
                      help='the number of jobs to run simultaneously')
    parser.add_option('-f', '--force', action="store_true", default=False,
                      help='regenerate the output even if it is new')
    (options, args) = parser.parse_args()

    if options.jobs:
        try:
            jobs = int(options.jobs)
        except ValueError:
            sys.exit('Option -j (--jobs) takes a number')
    else:
        try:
            jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
                                     stdout=subprocess.PIPE).communicate()[0])
        except (OSError, ValueError):
            print 'info: failed to get the number of CPUs. Set jobs to 1'
            jobs = 1

    gen_boards_cfg(jobs, force=options.force)

if __name__ == '__main__':
    main()