tools/genboardscfg.py

   1 #!/usr/bin/env python2
   2 #
   3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Converter from Kconfig and MAINTAINERS to a board database.
  10
  11 Run 'tools/genboardscfg.py' to create a board database.
  12
  13 Run 'tools/genboardscfg.py -h' for available options.
  14
  15 Python 2.6 or later, but not Python 3.x is necessary to run this script.
  16 """
  17
  18 import errno
  19 import fnmatch
  20 import glob
  21 import multiprocessing
  22 import optparse
  23 import os
  24 import subprocess
  25 import sys
  26 import tempfile
  27 import time
  28
  29 sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
  30 import kconfiglib
  31
  32 ### constant variables ###
  33 OUTPUT_FILE = 'boards.cfg'
  34 CONFIG_DIR = 'configs'
  35 SLEEP_TIME = 0.03
  36 COMMENT_BLOCK = '''#
  37 # List of boards
  38 #   Automatically generated by %s: don't edit
  39 #
  40 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
  41
  42 ''' % __file__
  43
  44 ### helper functions ###
  45 def try_remove(f):
  46     """Remove a file ignoring 'No such file or directory' error."""
  47     try:
  48         os.remove(f)
  49     except OSError as exception:
  50         # Ignore 'No such file or directory' error
  51         if exception.errno != errno.ENOENT:
  52             raise
  53
  54 def check_top_directory():
  55     """Exit if we are not at the top of source directory."""
  56     for f in ('README', 'Licenses'):
  57         if not os.path.exists(f):
  58             sys.exit('Please run at the top of source directory.')
  59
  60 def output_is_new(output):
  61     """Check if the output file is up to date.
  62
  63     Returns:
  64       True if the given output file exists and is newer than any of
  65       *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
  66     """
  67     try:
  68         ctime = os.path.getctime(output)
  69     except OSError as exception:
  70         if exception.errno == errno.ENOENT:
  71             # return False on 'No such file or directory' error
  72             return False
  73         else:
  74             raise
  75
  76     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
  77         for filename in fnmatch.filter(filenames, '*_defconfig'):
  78             if fnmatch.fnmatch(filename, '.*'):
  79                 continue
  80             filepath = os.path.join(dirpath, filename)
  81             if ctime < os.path.getctime(filepath):
  82                 return False
  83
  84     for (dirpath, dirnames, filenames) in os.walk('.'):
  85         for filename in filenames:
  86             if (fnmatch.fnmatch(filename, '*~') or
  87                 not fnmatch.fnmatch(filename, 'Kconfig*') and
  88                 not filename == 'MAINTAINERS'):
  89                 continue
  90             filepath = os.path.join(dirpath, filename)
  91             if ctime < os.path.getctime(filepath):
  92                 return False
  93
  94     # Detect a board that has been removed since the current board database
  95     # was generated
  96     with open(output) as f:
  97         for line in f:
  98             if line[0] == '#' or line == '\n':
  99                 continue
 100             defconfig = line.split()[6] + '_defconfig'
 101             if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
 102                 return False
 103
 104     return True
 105
 106 ### classes ###
 107 class KconfigScanner:
 108
 109     """Kconfig scanner."""
 110
 111     ### constant variable only used in this class ###
 112     _SYMBOL_TABLE = {
 113         'arch' : 'SYS_ARCH',
 114         'cpu' : 'SYS_CPU',
 115         'soc' : 'SYS_SOC',
 116         'vendor' : 'SYS_VENDOR',
 117         'board' : 'SYS_BOARD',
 118         'config' : 'SYS_CONFIG_NAME',
 119         'options' : 'SYS_EXTRA_OPTIONS'
 120     }
 121
 122     def __init__(self):
 123         """Scan all the Kconfig files and create a Config object."""
 124         # Define environment variables referenced from Kconfig
 125         os.environ['srctree'] = os.getcwd()
 126         os.environ['UBOOTVERSION'] = 'dummy'
 127         os.environ['KCONFIG_OBJDIR'] = ''
 128         self._conf = kconfiglib.Config()
 129
 130     def __del__(self):
 131         """Delete a leftover temporary file before exit.
 132
 133         The scan() method of this class creates a temporay file and deletes
 134         it on success.  If scan() method throws an exception on the way,
 135         the temporary file might be left over.  In that case, it should be
 136         deleted in this destructor.
 137         """
 138         if hasattr(self, '_tmpfile') and self._tmpfile:
 139             try_remove(self._tmpfile)
 140
 141     def scan(self, defconfig):
 142         """Load a defconfig file to obtain board parameters.
 143
 144         Arguments:
 145           defconfig: path to the defconfig file to be processed
 146
 147         Returns:
 148           A dictionary of board parameters.  It has a form of:
 149           {
 150               'arch': <arch_name>,
 151               'cpu': <cpu_name>,
 152               'soc': <soc_name>,
 153               'vendor': <vendor_name>,
 154               'board': <board_name>,
 155               'target': <target_name>,
 156               'config': <config_header_name>,
 157               'options': <extra_options>
 158           }
 159         """
 160         # strip special prefixes and save it in a temporary file
 161         fd, self._tmpfile = tempfile.mkstemp()
 162         with os.fdopen(fd, 'w') as f:
 163             for line in open(defconfig):
 164                 colon = line.find(':CONFIG_')
 165                 if colon == -1:
 166                     f.write(line)
 167                 else:
 168                     f.write(line[colon + 1:])
 169
 170         self._conf.load_config(self._tmpfile)
 171
 172         try_remove(self._tmpfile)
 173         self._tmpfile = None
 174
 175         params = {}
 176
 177         # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
 178         # Set '-' if the value is empty.
 179         for key, symbol in self._SYMBOL_TABLE.items():
 180             value = self._conf.get_symbol(symbol).get_value()
 181             if value:
 182                 params[key] = value
 183             else:
 184                 params[key] = '-'
 185
 186         defconfig = os.path.basename(defconfig)
 187         params['target'], match, rear = defconfig.partition('_defconfig')
 188         assert match and not rear, '%s : invalid defconfig' % defconfig
 189
 190         # fix-up for aarch64
 191         if params['arch'] == 'arm' and params['cpu'] == 'armv8':
 192             params['arch'] = 'aarch64'
 193
 194         # fix-up options field. It should have the form:
 195         # <config name>[:comma separated config options]
 196         if params['options'] != '-':
 197             params['options'] = params['config'] + ':' + \
 198                                 params['options'].replace(r'\"', '"')
 199         elif params['config'] != params['target']:
 200             params['options'] = params['config']
 201
 202         return params
 203
 204 def scan_defconfigs_for_multiprocess(queue, defconfigs):
 205     """Scan defconfig files and queue their board parameters
 206
 207     This function is intended to be passed to
 208     multiprocessing.Process() constructor.
 209
 210     Arguments:
 211       queue: An instance of multiprocessing.Queue().
 212              The resulting board parameters are written into it.
 213       defconfigs: A sequence of defconfig files to be scanned.
 214     """
 215     kconf_scanner = KconfigScanner()
 216     for defconfig in defconfigs:
 217         queue.put(kconf_scanner.scan(defconfig))
 218
 219 def read_queues(queues, params_list):
 220     """Read the queues and append the data to the paramers list"""
 221     for q in queues:
 222         while not q.empty():
 223             params_list.append(q.get())
 224
 225 def scan_defconfigs(jobs=1):
 226     """Collect board parameters for all defconfig files.
 227
 228     This function invokes multiple processes for faster processing.
 229
 230     Arguments:
 231       jobs: The number of jobs to run simultaneously
 232     """
 233     all_defconfigs = []
 234     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 235         for filename in fnmatch.filter(filenames, '*_defconfig'):
 236             if fnmatch.fnmatch(filename, '.*'):
 237                 continue
 238             all_defconfigs.append(os.path.join(dirpath, filename))
 239
 240     total_boards = len(all_defconfigs)
 241     processes = []
 242     queues = []
 243     for i in range(jobs):
 244         defconfigs = all_defconfigs[total_boards * i / jobs :
 245                                     total_boards * (i + 1) / jobs]
 246         q = multiprocessing.Queue(maxsize=-1)
 247         p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
 248                                     args=(q, defconfigs))
 249         p.start()
 250         processes.append(p)
 251         queues.append(q)
 252
 253     # The resulting data should be accumulated to this list
 254     params_list = []
 255
 256     # Data in the queues should be retrieved preriodically.
 257     # Otherwise, the queues would become full and subprocesses would get stuck.
 258     while any([p.is_alive() for p in processes]):
 259         read_queues(queues, params_list)
 260         # sleep for a while until the queues are filled
 261         time.sleep(SLEEP_TIME)
 262
 263     # Joining subprocesses just in case
 264     # (All subprocesses should already have been finished)
 265     for p in processes:
 266         p.join()
 267
 268     # retrieve leftover data
 269     read_queues(queues, params_list)
 270
 271     return params_list
 272
 273 class MaintainersDatabase:
 274
 275     """The database of board status and maintainers."""
 276
 277     def __init__(self):
 278         """Create an empty database."""
 279         self.database = {}
 280
 281     def get_status(self, target):
 282         """Return the status of the given board.
 283
 284         The board status is generally either 'Active' or 'Orphan'.
 285         Display a warning message and return '-' if status information
 286         is not found.
 287
 288         Returns:
 289           'Active', 'Orphan' or '-'.
 290         """
 291         if not target in self.database:
 292             print >> sys.stderr, "WARNING: no status info for '%s'" % target
 293             return '-'
 294
 295         tmp = self.database[target][0]
 296         if tmp.startswith('Maintained'):
 297             return 'Active'
 298         elif tmp.startswith('Orphan'):
 299             return 'Orphan'
 300         else:
 301             print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
 302                                   (tmp, target))
 303             return '-'
 304
 305     def get_maintainers(self, target):
 306         """Return the maintainers of the given board.
 307
 308         Returns:
 309           Maintainers of the board.  If the board has two or more maintainers,
 310           they are separated with colons.
 311         """
 312         if not target in self.database:
 313             print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
 314             return ''
 315
 316         return ':'.join(self.database[target][1])
 317
 318     def parse_file(self, file):
 319         """Parse a MAINTAINERS file.
 320
 321         Parse a MAINTAINERS file and accumulates board status and
 322         maintainers information.
 323
 324         Arguments:
 325           file: MAINTAINERS file to be parsed
 326         """
 327         targets = []
 328         maintainers = []
 329         status = '-'
 330         for line in open(file):
 331             # Check also commented maintainers
 332             if line[:3] == '#M:':
 333                 line = line[1:]
 334             tag, rest = line[:2], line[2:].strip()
 335             if tag == 'M:':
 336                 maintainers.append(rest)
 337             elif tag == 'F:':
 338                 # expand wildcard and filter by 'configs/*_defconfig'
 339                 for f in glob.glob(rest):
 340                     front, match, rear = f.partition('configs/')
 341                     if not front and match:
 342                         front, match, rear = rear.rpartition('_defconfig')
 343                         if match and not rear:
 344                             targets.append(front)
 345             elif tag == 'S:':
 346                 status = rest
 347             elif line == '\n':
 348                 for target in targets:
 349                     self.database[target] = (status, maintainers)
 350                 targets = []
 351                 maintainers = []
 352                 status = '-'
 353         if targets:
 354             for target in targets:
 355                 self.database[target] = (status, maintainers)
 356
 357 def insert_maintainers_info(params_list):
 358     """Add Status and Maintainers information to the board parameters list.
 359
 360     Arguments:
 361       params_list: A list of the board parameters
 362     """
 363     database = MaintainersDatabase()
 364     for (dirpath, dirnames, filenames) in os.walk('.'):
 365         if 'MAINTAINERS' in filenames:
 366             database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
 367
 368     for i, params in enumerate(params_list):
 369         target = params['target']
 370         params['status'] = database.get_status(target)
 371         params['maintainers'] = database.get_maintainers(target)
 372         params_list[i] = params
 373
 374 def format_and_output(params_list, output):
 375     """Write board parameters into a file.
 376
 377     Columnate the board parameters, sort lines alphabetically,
 378     and then write them to a file.
 379
 380     Arguments:
 381       params_list: The list of board parameters
 382       output: The path to the output file
 383     """
 384     FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
 385               'options', 'maintainers')
 386
 387     # First, decide the width of each column
 388     max_length = dict([ (f, 0) for f in FIELDS])
 389     for params in params_list:
 390         for f in FIELDS:
 391             max_length[f] = max(max_length[f], len(params[f]))
 392
 393     output_lines = []
 394     for params in params_list:
 395         line = ''
 396         for f in FIELDS:
 397             # insert two spaces between fields like column -t would
 398             line += '  ' + params[f].ljust(max_length[f])
 399         output_lines.append(line.strip())
 400
 401     # ignore case when sorting
 402     output_lines.sort(key=str.lower)
 403
 404     with open(output, 'w') as f:
 405         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 406
 407 def gen_boards_cfg(output, jobs=1, force=False):
 408     """Generate a board database file.
 409
 410     Arguments:
 411       output: The name of the output file
 412       jobs: The number of jobs to run simultaneously
 413       force: Force to generate the output even if it is new
 414     """
 415     check_top_directory()
 416
 417     if not force and output_is_new(output):
 418         print "%s is up to date. Nothing to do." % output
 419         sys.exit(0)
 420
 421     params_list = scan_defconfigs(jobs)
 422     insert_maintainers_info(params_list)
 423     format_and_output(params_list, output)
 424
 425 def main():
 426     try:
 427         cpu_count = multiprocessing.cpu_count()
 428     except NotImplementedError:
 429         cpu_count = 1
 430
 431     parser = optparse.OptionParser()
 432     # Add options here
 433     parser.add_option('-f', '--force', action="store_true", default=False,
 434                       help='regenerate the output even if it is new')
 435     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 436                       help='the number of jobs to run simultaneously')
 437     parser.add_option('-o', '--output', default=OUTPUT_FILE,
 438                       help='output file [default=%s]' % OUTPUT_FILE)
 439     (options, args) = parser.parse_args()
 440
 441     gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
 442
 443 if __name__ == '__main__':
 444     main()