tools/genboardscfg.py

   1 #!/usr/bin/env python2
   2 # SPDX-License-Identifier: GPL-2.0+
   3 #
   4 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   5 #
   6
   7 """
   8 Converter from Kconfig and MAINTAINERS to a board database.
   9
  10 Run 'tools/genboardscfg.py' to create a board database.
  11
  12 Run 'tools/genboardscfg.py -h' for available options.
  13
  14 Python 2.6 or later, but not Python 3.x is necessary to run this script.
  15 """
  16
  17 import errno
  18 import fnmatch
  19 import glob
  20 import multiprocessing
  21 import optparse
  22 import os
  23 import sys
  24 import tempfile
  25 import time
  26
  27 sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'buildman'))
  28 import kconfiglib
  29
  30 ### constant variables ###
  31 OUTPUT_FILE = 'boards.cfg'
  32 CONFIG_DIR = 'configs'
  33 SLEEP_TIME = 0.03
  34 COMMENT_BLOCK = '''#
  35 # List of boards
  36 #   Automatically generated by %s: don't edit
  37 #
  38 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
  39
  40 ''' % __file__
  41
  42 ### helper functions ###
  43 def try_remove(f):
  44     """Remove a file ignoring 'No such file or directory' error."""
  45     try:
  46         os.remove(f)
  47     except OSError as exception:
  48         # Ignore 'No such file or directory' error
  49         if exception.errno != errno.ENOENT:
  50             raise
  51
  52 def check_top_directory():
  53     """Exit if we are not at the top of source directory."""
  54     for f in ('README', 'Licenses'):
  55         if not os.path.exists(f):
  56             sys.exit('Please run at the top of source directory.')
  57
  58 def output_is_new(output):
  59     """Check if the output file is up to date.
  60
  61     Returns:
  62       True if the given output file exists and is newer than any of
  63       *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
  64     """
  65     try:
  66         ctime = os.path.getctime(output)
  67     except OSError as exception:
  68         if exception.errno == errno.ENOENT:
  69             # return False on 'No such file or directory' error
  70             return False
  71         else:
  72             raise
  73
  74     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
  75         for filename in fnmatch.filter(filenames, '*_defconfig'):
  76             if fnmatch.fnmatch(filename, '.*'):
  77                 continue
  78             filepath = os.path.join(dirpath, filename)
  79             if ctime < os.path.getctime(filepath):
  80                 return False
  81
  82     for (dirpath, dirnames, filenames) in os.walk('.'):
  83         for filename in filenames:
  84             if (fnmatch.fnmatch(filename, '*~') or
  85                 not fnmatch.fnmatch(filename, 'Kconfig*') and
  86                 not filename == 'MAINTAINERS'):
  87                 continue
  88             filepath = os.path.join(dirpath, filename)
  89             if ctime < os.path.getctime(filepath):
  90                 return False
  91
  92     # Detect a board that has been removed since the current board database
  93     # was generated
  94     with open(output) as f:
  95         for line in f:
  96             if line[0] == '#' or line == '\n':
  97                 continue
  98             defconfig = line.split()[6] + '_defconfig'
  99             if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
 100                 return False
 101
 102     return True
 103
 104 ### classes ###
 105 class KconfigScanner:
 106
 107     """Kconfig scanner."""
 108
 109     ### constant variable only used in this class ###
 110     _SYMBOL_TABLE = {
 111         'arch' : 'SYS_ARCH',
 112         'cpu' : 'SYS_CPU',
 113         'soc' : 'SYS_SOC',
 114         'vendor' : 'SYS_VENDOR',
 115         'board' : 'SYS_BOARD',
 116         'config' : 'SYS_CONFIG_NAME',
 117         'options' : 'SYS_EXTRA_OPTIONS'
 118     }
 119
 120     def __init__(self):
 121         """Scan all the Kconfig files and create a Config object."""
 122         # Define environment variables referenced from Kconfig
 123         os.environ['srctree'] = os.getcwd()
 124         os.environ['UBOOTVERSION'] = 'dummy'
 125         os.environ['KCONFIG_OBJDIR'] = ''
 126         self._conf = kconfiglib.Config(print_warnings=False)
 127
 128     def __del__(self):
 129         """Delete a leftover temporary file before exit.
 130
 131         The scan() method of this class creates a temporay file and deletes
 132         it on success.  If scan() method throws an exception on the way,
 133         the temporary file might be left over.  In that case, it should be
 134         deleted in this destructor.
 135         """
 136         if hasattr(self, '_tmpfile') and self._tmpfile:
 137             try_remove(self._tmpfile)
 138
 139     def scan(self, defconfig):
 140         """Load a defconfig file to obtain board parameters.
 141
 142         Arguments:
 143           defconfig: path to the defconfig file to be processed
 144
 145         Returns:
 146           A dictionary of board parameters.  It has a form of:
 147           {
 148               'arch': <arch_name>,
 149               'cpu': <cpu_name>,
 150               'soc': <soc_name>,
 151               'vendor': <vendor_name>,
 152               'board': <board_name>,
 153               'target': <target_name>,
 154               'config': <config_header_name>,
 155               'options': <extra_options>
 156           }
 157         """
 158         # strip special prefixes and save it in a temporary file
 159         fd, self._tmpfile = tempfile.mkstemp()
 160         with os.fdopen(fd, 'w') as f:
 161             for line in open(defconfig):
 162                 colon = line.find(':CONFIG_')
 163                 if colon == -1:
 164                     f.write(line)
 165                 else:
 166                     f.write(line[colon + 1:])
 167
 168         warnings = self._conf.load_config(self._tmpfile)
 169         if warnings:
 170             for warning in warnings:
 171                 print '%s: %s' % (defconfig, warning)
 172
 173         try_remove(self._tmpfile)
 174         self._tmpfile = None
 175
 176         params = {}
 177
 178         # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
 179         # Set '-' if the value is empty.
 180         for key, symbol in self._SYMBOL_TABLE.items():
 181             value = self._conf.get_symbol(symbol).get_value()
 182             if value:
 183                 params[key] = value
 184             else:
 185                 params[key] = '-'
 186
 187         defconfig = os.path.basename(defconfig)
 188         params['target'], match, rear = defconfig.partition('_defconfig')
 189         assert match and not rear, '%s : invalid defconfig' % defconfig
 190
 191         # fix-up for aarch64
 192         if params['arch'] == 'arm' and params['cpu'] == 'armv8':
 193             params['arch'] = 'aarch64'
 194
 195         # fix-up options field. It should have the form:
 196         # <config name>[:comma separated config options]
 197         if params['options'] != '-':
 198             params['options'] = params['config'] + ':' + \
 199                                 params['options'].replace(r'\"', '"')
 200         elif params['config'] != params['target']:
 201             params['options'] = params['config']
 202
 203         return params
 204
 205 def scan_defconfigs_for_multiprocess(queue, defconfigs):
 206     """Scan defconfig files and queue their board parameters
 207
 208     This function is intended to be passed to
 209     multiprocessing.Process() constructor.
 210
 211     Arguments:
 212       queue: An instance of multiprocessing.Queue().
 213              The resulting board parameters are written into it.
 214       defconfigs: A sequence of defconfig files to be scanned.
 215     """
 216     kconf_scanner = KconfigScanner()
 217     for defconfig in defconfigs:
 218         queue.put(kconf_scanner.scan(defconfig))
 219
 220 def read_queues(queues, params_list):
 221     """Read the queues and append the data to the paramers list"""
 222     for q in queues:
 223         while not q.empty():
 224             params_list.append(q.get())
 225
 226 def scan_defconfigs(jobs=1):
 227     """Collect board parameters for all defconfig files.
 228
 229     This function invokes multiple processes for faster processing.
 230
 231     Arguments:
 232       jobs: The number of jobs to run simultaneously
 233     """
 234     all_defconfigs = []
 235     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 236         for filename in fnmatch.filter(filenames, '*_defconfig'):
 237             if fnmatch.fnmatch(filename, '.*'):
 238                 continue
 239             all_defconfigs.append(os.path.join(dirpath, filename))
 240
 241     total_boards = len(all_defconfigs)
 242     processes = []
 243     queues = []
 244     for i in range(jobs):
 245         defconfigs = all_defconfigs[total_boards * i / jobs :
 246                                     total_boards * (i + 1) / jobs]
 247         q = multiprocessing.Queue(maxsize=-1)
 248         p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
 249                                     args=(q, defconfigs))
 250         p.start()
 251         processes.append(p)
 252         queues.append(q)
 253
 254     # The resulting data should be accumulated to this list
 255     params_list = []
 256
 257     # Data in the queues should be retrieved preriodically.
 258     # Otherwise, the queues would become full and subprocesses would get stuck.
 259     while any([p.is_alive() for p in processes]):
 260         read_queues(queues, params_list)
 261         # sleep for a while until the queues are filled
 262         time.sleep(SLEEP_TIME)
 263
 264     # Joining subprocesses just in case
 265     # (All subprocesses should already have been finished)
 266     for p in processes:
 267         p.join()
 268
 269     # retrieve leftover data
 270     read_queues(queues, params_list)
 271
 272     return params_list
 273
 274 class MaintainersDatabase:
 275
 276     """The database of board status and maintainers."""
 277
 278     def __init__(self):
 279         """Create an empty database."""
 280         self.database = {}
 281
 282     def get_status(self, target):
 283         """Return the status of the given board.
 284
 285         The board status is generally either 'Active' or 'Orphan'.
 286         Display a warning message and return '-' if status information
 287         is not found.
 288
 289         Returns:
 290           'Active', 'Orphan' or '-'.
 291         """
 292         if not target in self.database:
 293             print >> sys.stderr, "WARNING: no status info for '%s'" % target
 294             return '-'
 295
 296         tmp = self.database[target][0]
 297         if tmp.startswith('Maintained'):
 298             return 'Active'
 299         elif tmp.startswith('Supported'):
 300             return 'Active'
 301         elif tmp.startswith('Orphan'):
 302             return 'Orphan'
 303         else:
 304             print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
 305                                   (tmp, target))
 306             return '-'
 307
 308     def get_maintainers(self, target):
 309         """Return the maintainers of the given board.
 310
 311         Returns:
 312           Maintainers of the board.  If the board has two or more maintainers,
 313           they are separated with colons.
 314         """
 315         if not target in self.database:
 316             print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
 317             return ''
 318
 319         return ':'.join(self.database[target][1])
 320
 321     def parse_file(self, file):
 322         """Parse a MAINTAINERS file.
 323
 324         Parse a MAINTAINERS file and accumulates board status and
 325         maintainers information.
 326
 327         Arguments:
 328           file: MAINTAINERS file to be parsed
 329         """
 330         targets = []
 331         maintainers = []
 332         status = '-'
 333         for line in open(file):
 334             # Check also commented maintainers
 335             if line[:3] == '#M:':
 336                 line = line[1:]
 337             tag, rest = line[:2], line[2:].strip()
 338             if tag == 'M:':
 339                 maintainers.append(rest)
 340             elif tag == 'F:':
 341                 # expand wildcard and filter by 'configs/*_defconfig'
 342                 for f in glob.glob(rest):
 343                     front, match, rear = f.partition('configs/')
 344                     if not front and match:
 345                         front, match, rear = rear.rpartition('_defconfig')
 346                         if match and not rear:
 347                             targets.append(front)
 348             elif tag == 'S:':
 349                 status = rest
 350             elif line == '\n':
 351                 for target in targets:
 352                     self.database[target] = (status, maintainers)
 353                 targets = []
 354                 maintainers = []
 355                 status = '-'
 356         if targets:
 357             for target in targets:
 358                 self.database[target] = (status, maintainers)
 359
 360 def insert_maintainers_info(params_list):
 361     """Add Status and Maintainers information to the board parameters list.
 362
 363     Arguments:
 364       params_list: A list of the board parameters
 365     """
 366     database = MaintainersDatabase()
 367     for (dirpath, dirnames, filenames) in os.walk('.'):
 368         if 'MAINTAINERS' in filenames:
 369             database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
 370
 371     for i, params in enumerate(params_list):
 372         target = params['target']
 373         params['status'] = database.get_status(target)
 374         params['maintainers'] = database.get_maintainers(target)
 375         params_list[i] = params
 376
 377 def format_and_output(params_list, output):
 378     """Write board parameters into a file.
 379
 380     Columnate the board parameters, sort lines alphabetically,
 381     and then write them to a file.
 382
 383     Arguments:
 384       params_list: The list of board parameters
 385       output: The path to the output file
 386     """
 387     FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
 388               'options', 'maintainers')
 389
 390     # First, decide the width of each column
 391     max_length = dict([ (f, 0) for f in FIELDS])
 392     for params in params_list:
 393         for f in FIELDS:
 394             max_length[f] = max(max_length[f], len(params[f]))
 395
 396     output_lines = []
 397     for params in params_list:
 398         line = ''
 399         for f in FIELDS:
 400             # insert two spaces between fields like column -t would
 401             line += '  ' + params[f].ljust(max_length[f])
 402         output_lines.append(line.strip())
 403
 404     # ignore case when sorting
 405     output_lines.sort(key=str.lower)
 406
 407     with open(output, 'w') as f:
 408         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 409
 410 def gen_boards_cfg(output, jobs=1, force=False):
 411     """Generate a board database file.
 412
 413     Arguments:
 414       output: The name of the output file
 415       jobs: The number of jobs to run simultaneously
 416       force: Force to generate the output even if it is new
 417     """
 418     check_top_directory()
 419
 420     if not force and output_is_new(output):
 421         print "%s is up to date. Nothing to do." % output
 422         sys.exit(0)
 423
 424     params_list = scan_defconfigs(jobs)
 425     insert_maintainers_info(params_list)
 426     format_and_output(params_list, output)
 427
 428 def main():
 429     try:
 430         cpu_count = multiprocessing.cpu_count()
 431     except NotImplementedError:
 432         cpu_count = 1
 433
 434     parser = optparse.OptionParser()
 435     # Add options here
 436     parser.add_option('-f', '--force', action="store_true", default=False,
 437                       help='regenerate the output even if it is new')
 438     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 439                       help='the number of jobs to run simultaneously')
 440     parser.add_option('-o', '--output', default=OUTPUT_FILE,
 441                       help='output file [default=%s]' % OUTPUT_FILE)
 442     (options, args) = parser.parse_args()
 443
 444     gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
 445
 446 if __name__ == '__main__':
 447     main()