tools/genboardscfg.py

   1 #!/usr/bin/env python2
   2 #
   3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Converter from Kconfig and MAINTAINERS to a board database.
  10
  11 Run 'tools/genboardscfg.py' to create a board database.
  12
  13 Run 'tools/genboardscfg.py -h' for available options.
  14
  15 Python 2.6 or later, but not Python 3.x is necessary to run this script.
  16 """
  17
  18 import errno
  19 import fnmatch
  20 import glob
  21 import multiprocessing
  22 import optparse
  23 import os
  24 import sys
  25 import tempfile
  26 import time
  27
  28 sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
  29 import kconfiglib
  30
  31 ### constant variables ###
  32 OUTPUT_FILE = 'boards.cfg'
  33 CONFIG_DIR = 'configs'
  34 SLEEP_TIME = 0.03
  35 COMMENT_BLOCK = '''#
  36 # List of boards
  37 #   Automatically generated by %s: don't edit
  38 #
  39 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
  40
  41 ''' % __file__
  42
  43 ### helper functions ###
  44 def try_remove(f):
  45     """Remove a file ignoring 'No such file or directory' error."""
  46     try:
  47         os.remove(f)
  48     except OSError as exception:
  49         # Ignore 'No such file or directory' error
  50         if exception.errno != errno.ENOENT:
  51             raise
  52
  53 def check_top_directory():
  54     """Exit if we are not at the top of source directory."""
  55     for f in ('README', 'Licenses'):
  56         if not os.path.exists(f):
  57             sys.exit('Please run at the top of source directory.')
  58
  59 def output_is_new(output):
  60     """Check if the output file is up to date.
  61
  62     Returns:
  63       True if the given output file exists and is newer than any of
  64       *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
  65     """
  66     try:
  67         ctime = os.path.getctime(output)
  68     except OSError as exception:
  69         if exception.errno == errno.ENOENT:
  70             # return False on 'No such file or directory' error
  71             return False
  72         else:
  73             raise
  74
  75     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
  76         for filename in fnmatch.filter(filenames, '*_defconfig'):
  77             if fnmatch.fnmatch(filename, '.*'):
  78                 continue
  79             filepath = os.path.join(dirpath, filename)
  80             if ctime < os.path.getctime(filepath):
  81                 return False
  82
  83     for (dirpath, dirnames, filenames) in os.walk('.'):
  84         for filename in filenames:
  85             if (fnmatch.fnmatch(filename, '*~') or
  86                 not fnmatch.fnmatch(filename, 'Kconfig*') and
  87                 not filename == 'MAINTAINERS'):
  88                 continue
  89             filepath = os.path.join(dirpath, filename)
  90             if ctime < os.path.getctime(filepath):
  91                 return False
  92
  93     # Detect a board that has been removed since the current board database
  94     # was generated
  95     with open(output) as f:
  96         for line in f:
  97             if line[0] == '#' or line == '\n':
  98                 continue
  99             defconfig = line.split()[6] + '_defconfig'
 100             if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
 101                 return False
 102
 103     return True
 104
 105 ### classes ###
 106 class KconfigScanner:
 107
 108     """Kconfig scanner."""
 109
 110     ### constant variable only used in this class ###
 111     _SYMBOL_TABLE = {
 112         'arch' : 'SYS_ARCH',
 113         'cpu' : 'SYS_CPU',
 114         'soc' : 'SYS_SOC',
 115         'vendor' : 'SYS_VENDOR',
 116         'board' : 'SYS_BOARD',
 117         'config' : 'SYS_CONFIG_NAME',
 118         'options' : 'SYS_EXTRA_OPTIONS'
 119     }
 120
 121     def __init__(self):
 122         """Scan all the Kconfig files and create a Config object."""
 123         # Define environment variables referenced from Kconfig
 124         os.environ['srctree'] = os.getcwd()
 125         os.environ['UBOOTVERSION'] = 'dummy'
 126         os.environ['KCONFIG_OBJDIR'] = ''
 127         self._conf = kconfiglib.Config(print_warnings=False)
 128
 129     def __del__(self):
 130         """Delete a leftover temporary file before exit.
 131
 132         The scan() method of this class creates a temporay file and deletes
 133         it on success.  If scan() method throws an exception on the way,
 134         the temporary file might be left over.  In that case, it should be
 135         deleted in this destructor.
 136         """
 137         if hasattr(self, '_tmpfile') and self._tmpfile:
 138             try_remove(self._tmpfile)
 139
 140     def scan(self, defconfig):
 141         """Load a defconfig file to obtain board parameters.
 142
 143         Arguments:
 144           defconfig: path to the defconfig file to be processed
 145
 146         Returns:
 147           A dictionary of board parameters.  It has a form of:
 148           {
 149               'arch': <arch_name>,
 150               'cpu': <cpu_name>,
 151               'soc': <soc_name>,
 152               'vendor': <vendor_name>,
 153               'board': <board_name>,
 154               'target': <target_name>,
 155               'config': <config_header_name>,
 156               'options': <extra_options>
 157           }
 158         """
 159         # strip special prefixes and save it in a temporary file
 160         fd, self._tmpfile = tempfile.mkstemp()
 161         with os.fdopen(fd, 'w') as f:
 162             for line in open(defconfig):
 163                 colon = line.find(':CONFIG_')
 164                 if colon == -1:
 165                     f.write(line)
 166                 else:
 167                     f.write(line[colon + 1:])
 168
 169         warnings = self._conf.load_config(self._tmpfile)
 170         if warnings:
 171             for warning in warnings:
 172                 print '%s: %s' % (defconfig, warning)
 173
 174         try_remove(self._tmpfile)
 175         self._tmpfile = None
 176
 177         params = {}
 178
 179         # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
 180         # Set '-' if the value is empty.
 181         for key, symbol in self._SYMBOL_TABLE.items():
 182             value = self._conf.get_symbol(symbol).get_value()
 183             if value:
 184                 params[key] = value
 185             else:
 186                 params[key] = '-'
 187
 188         defconfig = os.path.basename(defconfig)
 189         params['target'], match, rear = defconfig.partition('_defconfig')
 190         assert match and not rear, '%s : invalid defconfig' % defconfig
 191
 192         # fix-up for aarch64
 193         if params['arch'] == 'arm' and params['cpu'] == 'armv8':
 194             params['arch'] = 'aarch64'
 195
 196         # fix-up options field. It should have the form:
 197         # <config name>[:comma separated config options]
 198         if params['options'] != '-':
 199             params['options'] = params['config'] + ':' + \
 200                                 params['options'].replace(r'\"', '"')
 201         elif params['config'] != params['target']:
 202             params['options'] = params['config']
 203
 204         return params
 205
 206 def scan_defconfigs_for_multiprocess(queue, defconfigs):
 207     """Scan defconfig files and queue their board parameters
 208
 209     This function is intended to be passed to
 210     multiprocessing.Process() constructor.
 211
 212     Arguments:
 213       queue: An instance of multiprocessing.Queue().
 214              The resulting board parameters are written into it.
 215       defconfigs: A sequence of defconfig files to be scanned.
 216     """
 217     kconf_scanner = KconfigScanner()
 218     for defconfig in defconfigs:
 219         queue.put(kconf_scanner.scan(defconfig))
 220
 221 def read_queues(queues, params_list):
 222     """Read the queues and append the data to the paramers list"""
 223     for q in queues:
 224         while not q.empty():
 225             params_list.append(q.get())
 226
 227 def scan_defconfigs(jobs=1):
 228     """Collect board parameters for all defconfig files.
 229
 230     This function invokes multiple processes for faster processing.
 231
 232     Arguments:
 233       jobs: The number of jobs to run simultaneously
 234     """
 235     all_defconfigs = []
 236     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 237         for filename in fnmatch.filter(filenames, '*_defconfig'):
 238             if fnmatch.fnmatch(filename, '.*'):
 239                 continue
 240             all_defconfigs.append(os.path.join(dirpath, filename))
 241
 242     total_boards = len(all_defconfigs)
 243     processes = []
 244     queues = []
 245     for i in range(jobs):
 246         defconfigs = all_defconfigs[total_boards * i / jobs :
 247                                     total_boards * (i + 1) / jobs]
 248         q = multiprocessing.Queue(maxsize=-1)
 249         p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
 250                                     args=(q, defconfigs))
 251         p.start()
 252         processes.append(p)
 253         queues.append(q)
 254
 255     # The resulting data should be accumulated to this list
 256     params_list = []
 257
 258     # Data in the queues should be retrieved preriodically.
 259     # Otherwise, the queues would become full and subprocesses would get stuck.
 260     while any([p.is_alive() for p in processes]):
 261         read_queues(queues, params_list)
 262         # sleep for a while until the queues are filled
 263         time.sleep(SLEEP_TIME)
 264
 265     # Joining subprocesses just in case
 266     # (All subprocesses should already have been finished)
 267     for p in processes:
 268         p.join()
 269
 270     # retrieve leftover data
 271     read_queues(queues, params_list)
 272
 273     return params_list
 274
 275 class MaintainersDatabase:
 276
 277     """The database of board status and maintainers."""
 278
 279     def __init__(self):
 280         """Create an empty database."""
 281         self.database = {}
 282
 283     def get_status(self, target):
 284         """Return the status of the given board.
 285
 286         The board status is generally either 'Active' or 'Orphan'.
 287         Display a warning message and return '-' if status information
 288         is not found.
 289
 290         Returns:
 291           'Active', 'Orphan' or '-'.
 292         """
 293         if not target in self.database:
 294             print >> sys.stderr, "WARNING: no status info for '%s'" % target
 295             return '-'
 296
 297         tmp = self.database[target][0]
 298         if tmp.startswith('Maintained'):
 299             return 'Active'
 300         elif tmp.startswith('Supported'):
 301             return 'Active'
 302         elif tmp.startswith('Orphan'):
 303             return 'Orphan'
 304         else:
 305             print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
 306                                   (tmp, target))
 307             return '-'
 308
 309     def get_maintainers(self, target):
 310         """Return the maintainers of the given board.
 311
 312         Returns:
 313           Maintainers of the board.  If the board has two or more maintainers,
 314           they are separated with colons.
 315         """
 316         if not target in self.database:
 317             print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
 318             return ''
 319
 320         return ':'.join(self.database[target][1])
 321
 322     def parse_file(self, file):
 323         """Parse a MAINTAINERS file.
 324
 325         Parse a MAINTAINERS file and accumulates board status and
 326         maintainers information.
 327
 328         Arguments:
 329           file: MAINTAINERS file to be parsed
 330         """
 331         targets = []
 332         maintainers = []
 333         status = '-'
 334         for line in open(file):
 335             # Check also commented maintainers
 336             if line[:3] == '#M:':
 337                 line = line[1:]
 338             tag, rest = line[:2], line[2:].strip()
 339             if tag == 'M:':
 340                 maintainers.append(rest)
 341             elif tag == 'F:':
 342                 # expand wildcard and filter by 'configs/*_defconfig'
 343                 for f in glob.glob(rest):
 344                     front, match, rear = f.partition('configs/')
 345                     if not front and match:
 346                         front, match, rear = rear.rpartition('_defconfig')
 347                         if match and not rear:
 348                             targets.append(front)
 349             elif tag == 'S:':
 350                 status = rest
 351             elif line == '\n':
 352                 for target in targets:
 353                     self.database[target] = (status, maintainers)
 354                 targets = []
 355                 maintainers = []
 356                 status = '-'
 357         if targets:
 358             for target in targets:
 359                 self.database[target] = (status, maintainers)
 360
 361 def insert_maintainers_info(params_list):
 362     """Add Status and Maintainers information to the board parameters list.
 363
 364     Arguments:
 365       params_list: A list of the board parameters
 366     """
 367     database = MaintainersDatabase()
 368     for (dirpath, dirnames, filenames) in os.walk('.'):
 369         if 'MAINTAINERS' in filenames:
 370             database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
 371
 372     for i, params in enumerate(params_list):
 373         target = params['target']
 374         params['status'] = database.get_status(target)
 375         params['maintainers'] = database.get_maintainers(target)
 376         params_list[i] = params
 377
 378 def format_and_output(params_list, output):
 379     """Write board parameters into a file.
 380
 381     Columnate the board parameters, sort lines alphabetically,
 382     and then write them to a file.
 383
 384     Arguments:
 385       params_list: The list of board parameters
 386       output: The path to the output file
 387     """
 388     FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
 389               'options', 'maintainers')
 390
 391     # First, decide the width of each column
 392     max_length = dict([ (f, 0) for f in FIELDS])
 393     for params in params_list:
 394         for f in FIELDS:
 395             max_length[f] = max(max_length[f], len(params[f]))
 396
 397     output_lines = []
 398     for params in params_list:
 399         line = ''
 400         for f in FIELDS:
 401             # insert two spaces between fields like column -t would
 402             line += '  ' + params[f].ljust(max_length[f])
 403         output_lines.append(line.strip())
 404
 405     # ignore case when sorting
 406     output_lines.sort(key=str.lower)
 407
 408     with open(output, 'w') as f:
 409         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 410
 411 def gen_boards_cfg(output, jobs=1, force=False):
 412     """Generate a board database file.
 413
 414     Arguments:
 415       output: The name of the output file
 416       jobs: The number of jobs to run simultaneously
 417       force: Force to generate the output even if it is new
 418     """
 419     check_top_directory()
 420
 421     if not force and output_is_new(output):
 422         print "%s is up to date. Nothing to do." % output
 423         sys.exit(0)
 424
 425     params_list = scan_defconfigs(jobs)
 426     insert_maintainers_info(params_list)
 427     format_and_output(params_list, output)
 428
 429 def main():
 430     try:
 431         cpu_count = multiprocessing.cpu_count()
 432     except NotImplementedError:
 433         cpu_count = 1
 434
 435     parser = optparse.OptionParser()
 436     # Add options here
 437     parser.add_option('-f', '--force', action="store_true", default=False,
 438                       help='regenerate the output even if it is new')
 439     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 440                       help='the number of jobs to run simultaneously')
 441     parser.add_option('-o', '--output', default=OUTPUT_FILE,
 442                       help='output file [default=%s]' % OUTPUT_FILE)
 443     (options, args) = parser.parse_args()
 444
 445     gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
 446
 447 if __name__ == '__main__':
 448     main()