tools/genboardscfg.py

   1 #!/usr/bin/env python2
   2 #
   3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Converter from Kconfig and MAINTAINERS to a board database.
  10
  11 Run 'tools/genboardscfg.py' to create a board database.
  12
  13 Run 'tools/genboardscfg.py -h' for available options.
  14
  15 Python 2.6 or later, but not Python 3.x is necessary to run this script.
  16 """
  17
  18 import errno
  19 import fnmatch
  20 import glob
  21 import multiprocessing
  22 import optparse
  23 import os
  24 import sys
  25 import tempfile
  26 import time
  27
  28 sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
  29 import kconfiglib
  30
  31 ### constant variables ###
  32 OUTPUT_FILE = 'boards.cfg'
  33 CONFIG_DIR = 'configs'
  34 SLEEP_TIME = 0.03
  35 COMMENT_BLOCK = '''#
  36 # List of boards
  37 #   Automatically generated by %s: don't edit
  38 #
  39 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
  40
  41 ''' % __file__
  42
  43 ### helper functions ###
  44 def try_remove(f):
  45     """Remove a file ignoring 'No such file or directory' error."""
  46     try:
  47         os.remove(f)
  48     except OSError as exception:
  49         # Ignore 'No such file or directory' error
  50         if exception.errno != errno.ENOENT:
  51             raise
  52
  53 def check_top_directory():
  54     """Exit if we are not at the top of source directory."""
  55     for f in ('README', 'Licenses'):
  56         if not os.path.exists(f):
  57             sys.exit('Please run at the top of source directory.')
  58
  59 def output_is_new(output):
  60     """Check if the output file is up to date.
  61
  62     Returns:
  63       True if the given output file exists and is newer than any of
  64       *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
  65     """
  66     try:
  67         ctime = os.path.getctime(output)
  68     except OSError as exception:
  69         if exception.errno == errno.ENOENT:
  70             # return False on 'No such file or directory' error
  71             return False
  72         else:
  73             raise
  74
  75     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
  76         for filename in fnmatch.filter(filenames, '*_defconfig'):
  77             if fnmatch.fnmatch(filename, '.*'):
  78                 continue
  79             filepath = os.path.join(dirpath, filename)
  80             if ctime < os.path.getctime(filepath):
  81                 return False
  82
  83     for (dirpath, dirnames, filenames) in os.walk('.'):
  84         for filename in filenames:
  85             if (fnmatch.fnmatch(filename, '*~') or
  86                 not fnmatch.fnmatch(filename, 'Kconfig*') and
  87                 not filename == 'MAINTAINERS'):
  88                 continue
  89             filepath = os.path.join(dirpath, filename)
  90             if ctime < os.path.getctime(filepath):
  91                 return False
  92
  93     # Detect a board that has been removed since the current board database
  94     # was generated
  95     with open(output) as f:
  96         for line in f:
  97             if line[0] == '#' or line == '\n':
  98                 continue
  99             defconfig = line.split()[6] + '_defconfig'
 100             if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
 101                 return False
 102
 103     return True
 104
 105 ### classes ###
 106 class KconfigScanner:
 107
 108     """Kconfig scanner."""
 109
 110     ### constant variable only used in this class ###
 111     _SYMBOL_TABLE = {
 112         'arch' : 'SYS_ARCH',
 113         'cpu' : 'SYS_CPU',
 114         'soc' : 'SYS_SOC',
 115         'vendor' : 'SYS_VENDOR',
 116         'board' : 'SYS_BOARD',
 117         'config' : 'SYS_CONFIG_NAME',
 118         'options' : 'SYS_EXTRA_OPTIONS'
 119     }
 120
 121     def __init__(self):
 122         """Scan all the Kconfig files and create a Config object."""
 123         # Define environment variables referenced from Kconfig
 124         os.environ['srctree'] = os.getcwd()
 125         os.environ['UBOOTVERSION'] = 'dummy'
 126         os.environ['KCONFIG_OBJDIR'] = ''
 127         self._conf = kconfiglib.Config()
 128
 129     def __del__(self):
 130         """Delete a leftover temporary file before exit.
 131
 132         The scan() method of this class creates a temporay file and deletes
 133         it on success.  If scan() method throws an exception on the way,
 134         the temporary file might be left over.  In that case, it should be
 135         deleted in this destructor.
 136         """
 137         if hasattr(self, '_tmpfile') and self._tmpfile:
 138             try_remove(self._tmpfile)
 139
 140     def scan(self, defconfig):
 141         """Load a defconfig file to obtain board parameters.
 142
 143         Arguments:
 144           defconfig: path to the defconfig file to be processed
 145
 146         Returns:
 147           A dictionary of board parameters.  It has a form of:
 148           {
 149               'arch': <arch_name>,
 150               'cpu': <cpu_name>,
 151               'soc': <soc_name>,
 152               'vendor': <vendor_name>,
 153               'board': <board_name>,
 154               'target': <target_name>,
 155               'config': <config_header_name>,
 156               'options': <extra_options>
 157           }
 158         """
 159         # strip special prefixes and save it in a temporary file
 160         fd, self._tmpfile = tempfile.mkstemp()
 161         with os.fdopen(fd, 'w') as f:
 162             for line in open(defconfig):
 163                 colon = line.find(':CONFIG_')
 164                 if colon == -1:
 165                     f.write(line)
 166                 else:
 167                     f.write(line[colon + 1:])
 168
 169         self._conf.load_config(self._tmpfile)
 170
 171         try_remove(self._tmpfile)
 172         self._tmpfile = None
 173
 174         params = {}
 175
 176         # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
 177         # Set '-' if the value is empty.
 178         for key, symbol in self._SYMBOL_TABLE.items():
 179             value = self._conf.get_symbol(symbol).get_value()
 180             if value:
 181                 params[key] = value
 182             else:
 183                 params[key] = '-'
 184
 185         defconfig = os.path.basename(defconfig)
 186         params['target'], match, rear = defconfig.partition('_defconfig')
 187         assert match and not rear, '%s : invalid defconfig' % defconfig
 188
 189         # fix-up for aarch64
 190         if params['arch'] == 'arm' and params['cpu'] == 'armv8':
 191             params['arch'] = 'aarch64'
 192
 193         # fix-up options field. It should have the form:
 194         # <config name>[:comma separated config options]
 195         if params['options'] != '-':
 196             params['options'] = params['config'] + ':' + \
 197                                 params['options'].replace(r'\"', '"')
 198         elif params['config'] != params['target']:
 199             params['options'] = params['config']
 200
 201         return params
 202
 203 def scan_defconfigs_for_multiprocess(queue, defconfigs):
 204     """Scan defconfig files and queue their board parameters
 205
 206     This function is intended to be passed to
 207     multiprocessing.Process() constructor.
 208
 209     Arguments:
 210       queue: An instance of multiprocessing.Queue().
 211              The resulting board parameters are written into it.
 212       defconfigs: A sequence of defconfig files to be scanned.
 213     """
 214     kconf_scanner = KconfigScanner()
 215     for defconfig in defconfigs:
 216         queue.put(kconf_scanner.scan(defconfig))
 217
 218 def read_queues(queues, params_list):
 219     """Read the queues and append the data to the paramers list"""
 220     for q in queues:
 221         while not q.empty():
 222             params_list.append(q.get())
 223
 224 def scan_defconfigs(jobs=1):
 225     """Collect board parameters for all defconfig files.
 226
 227     This function invokes multiple processes for faster processing.
 228
 229     Arguments:
 230       jobs: The number of jobs to run simultaneously
 231     """
 232     all_defconfigs = []
 233     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 234         for filename in fnmatch.filter(filenames, '*_defconfig'):
 235             if fnmatch.fnmatch(filename, '.*'):
 236                 continue
 237             all_defconfigs.append(os.path.join(dirpath, filename))
 238
 239     total_boards = len(all_defconfigs)
 240     processes = []
 241     queues = []
 242     for i in range(jobs):
 243         defconfigs = all_defconfigs[total_boards * i / jobs :
 244                                     total_boards * (i + 1) / jobs]
 245         q = multiprocessing.Queue(maxsize=-1)
 246         p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
 247                                     args=(q, defconfigs))
 248         p.start()
 249         processes.append(p)
 250         queues.append(q)
 251
 252     # The resulting data should be accumulated to this list
 253     params_list = []
 254
 255     # Data in the queues should be retrieved preriodically.
 256     # Otherwise, the queues would become full and subprocesses would get stuck.
 257     while any([p.is_alive() for p in processes]):
 258         read_queues(queues, params_list)
 259         # sleep for a while until the queues are filled
 260         time.sleep(SLEEP_TIME)
 261
 262     # Joining subprocesses just in case
 263     # (All subprocesses should already have been finished)
 264     for p in processes:
 265         p.join()
 266
 267     # retrieve leftover data
 268     read_queues(queues, params_list)
 269
 270     return params_list
 271
 272 class MaintainersDatabase:
 273
 274     """The database of board status and maintainers."""
 275
 276     def __init__(self):
 277         """Create an empty database."""
 278         self.database = {}
 279
 280     def get_status(self, target):
 281         """Return the status of the given board.
 282
 283         The board status is generally either 'Active' or 'Orphan'.
 284         Display a warning message and return '-' if status information
 285         is not found.
 286
 287         Returns:
 288           'Active', 'Orphan' or '-'.
 289         """
 290         if not target in self.database:
 291             print >> sys.stderr, "WARNING: no status info for '%s'" % target
 292             return '-'
 293
 294         tmp = self.database[target][0]
 295         if tmp.startswith('Maintained'):
 296             return 'Active'
 297         elif tmp.startswith('Supported'):
 298             return 'Active'
 299         elif tmp.startswith('Orphan'):
 300             return 'Orphan'
 301         else:
 302             print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
 303                                   (tmp, target))
 304             return '-'
 305
 306     def get_maintainers(self, target):
 307         """Return the maintainers of the given board.
 308
 309         Returns:
 310           Maintainers of the board.  If the board has two or more maintainers,
 311           they are separated with colons.
 312         """
 313         if not target in self.database:
 314             print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
 315             return ''
 316
 317         return ':'.join(self.database[target][1])
 318
 319     def parse_file(self, file):
 320         """Parse a MAINTAINERS file.
 321
 322         Parse a MAINTAINERS file and accumulates board status and
 323         maintainers information.
 324
 325         Arguments:
 326           file: MAINTAINERS file to be parsed
 327         """
 328         targets = []
 329         maintainers = []
 330         status = '-'
 331         for line in open(file):
 332             # Check also commented maintainers
 333             if line[:3] == '#M:':
 334                 line = line[1:]
 335             tag, rest = line[:2], line[2:].strip()
 336             if tag == 'M:':
 337                 maintainers.append(rest)
 338             elif tag == 'F:':
 339                 # expand wildcard and filter by 'configs/*_defconfig'
 340                 for f in glob.glob(rest):
 341                     front, match, rear = f.partition('configs/')
 342                     if not front and match:
 343                         front, match, rear = rear.rpartition('_defconfig')
 344                         if match and not rear:
 345                             targets.append(front)
 346             elif tag == 'S:':
 347                 status = rest
 348             elif line == '\n':
 349                 for target in targets:
 350                     self.database[target] = (status, maintainers)
 351                 targets = []
 352                 maintainers = []
 353                 status = '-'
 354         if targets:
 355             for target in targets:
 356                 self.database[target] = (status, maintainers)
 357
 358 def insert_maintainers_info(params_list):
 359     """Add Status and Maintainers information to the board parameters list.
 360
 361     Arguments:
 362       params_list: A list of the board parameters
 363     """
 364     database = MaintainersDatabase()
 365     for (dirpath, dirnames, filenames) in os.walk('.'):
 366         if 'MAINTAINERS' in filenames:
 367             database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
 368
 369     for i, params in enumerate(params_list):
 370         target = params['target']
 371         params['status'] = database.get_status(target)
 372         params['maintainers'] = database.get_maintainers(target)
 373         params_list[i] = params
 374
 375 def format_and_output(params_list, output):
 376     """Write board parameters into a file.
 377
 378     Columnate the board parameters, sort lines alphabetically,
 379     and then write them to a file.
 380
 381     Arguments:
 382       params_list: The list of board parameters
 383       output: The path to the output file
 384     """
 385     FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
 386               'options', 'maintainers')
 387
 388     # First, decide the width of each column
 389     max_length = dict([ (f, 0) for f in FIELDS])
 390     for params in params_list:
 391         for f in FIELDS:
 392             max_length[f] = max(max_length[f], len(params[f]))
 393
 394     output_lines = []
 395     for params in params_list:
 396         line = ''
 397         for f in FIELDS:
 398             # insert two spaces between fields like column -t would
 399             line += '  ' + params[f].ljust(max_length[f])
 400         output_lines.append(line.strip())
 401
 402     # ignore case when sorting
 403     output_lines.sort(key=str.lower)
 404
 405     with open(output, 'w') as f:
 406         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 407
 408 def gen_boards_cfg(output, jobs=1, force=False):
 409     """Generate a board database file.
 410
 411     Arguments:
 412       output: The name of the output file
 413       jobs: The number of jobs to run simultaneously
 414       force: Force to generate the output even if it is new
 415     """
 416     check_top_directory()
 417
 418     if not force and output_is_new(output):
 419         print "%s is up to date. Nothing to do." % output
 420         sys.exit(0)
 421
 422     params_list = scan_defconfigs(jobs)
 423     insert_maintainers_info(params_list)
 424     format_and_output(params_list, output)
 425
 426 def main():
 427     try:
 428         cpu_count = multiprocessing.cpu_count()
 429     except NotImplementedError:
 430         cpu_count = 1
 431
 432     parser = optparse.OptionParser()
 433     # Add options here
 434     parser.add_option('-f', '--force', action="store_true", default=False,
 435                       help='regenerate the output even if it is new')
 436     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 437                       help='the number of jobs to run simultaneously')
 438     parser.add_option('-o', '--output', default=OUTPUT_FILE,
 439                       help='output file [default=%s]' % OUTPUT_FILE)
 440     (options, args) = parser.parse_args()
 441
 442     gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
 443
 444 if __name__ == '__main__':
 445     main()