tools/genboardscfg.py

   1 #!/usr/bin/env python2
   2 #
   3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Converter from Kconfig and MAINTAINERS to a board database.
  10
  11 Run 'tools/genboardscfg.py' to create a board database.
  12
  13 Run 'tools/genboardscfg.py -h' for available options.
  14
  15 Python 2.6 or later, but not Python 3.x is necessary to run this script.
  16 """
  17
  18 import errno
  19 import fnmatch
  20 import glob
  21 import multiprocessing
  22 import optparse
  23 import os
  24 import sys
  25 import tempfile
  26 import time
  27
  28 sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
  29 import kconfiglib
  30
  31 ### constant variables ###
  32 OUTPUT_FILE = 'boards.cfg'
  33 CONFIG_DIR = 'configs'
  34 SLEEP_TIME = 0.03
  35 COMMENT_BLOCK = '''#
  36 # List of boards
  37 #   Automatically generated by %s: don't edit
  38 #
  39 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
  40
  41 ''' % __file__
  42
  43 ### helper functions ###
  44 def try_remove(f):
  45     """Remove a file ignoring 'No such file or directory' error."""
  46     try:
  47         os.remove(f)
  48     except OSError as exception:
  49         # Ignore 'No such file or directory' error
  50         if exception.errno != errno.ENOENT:
  51             raise
  52
  53 def check_top_directory():
  54     """Exit if we are not at the top of source directory."""
  55     for f in ('README', 'Licenses'):
  56         if not os.path.exists(f):
  57             sys.exit('Please run at the top of source directory.')
  58
  59 def output_is_new(output):
  60     """Check if the output file is up to date.
  61
  62     Returns:
  63       True if the given output file exists and is newer than any of
  64       *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
  65     """
  66     try:
  67         ctime = os.path.getctime(output)
  68     except OSError as exception:
  69         if exception.errno == errno.ENOENT:
  70             # return False on 'No such file or directory' error
  71             return False
  72         else:
  73             raise
  74
  75     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
  76         for filename in fnmatch.filter(filenames, '*_defconfig'):
  77             if fnmatch.fnmatch(filename, '.*'):
  78                 continue
  79             filepath = os.path.join(dirpath, filename)
  80             if ctime < os.path.getctime(filepath):
  81                 return False
  82
  83     for (dirpath, dirnames, filenames) in os.walk('.'):
  84         for filename in filenames:
  85             if (fnmatch.fnmatch(filename, '*~') or
  86                 not fnmatch.fnmatch(filename, 'Kconfig*') and
  87                 not filename == 'MAINTAINERS'):
  88                 continue
  89             filepath = os.path.join(dirpath, filename)
  90             if ctime < os.path.getctime(filepath):
  91                 return False
  92
  93     # Detect a board that has been removed since the current board database
  94     # was generated
  95     with open(output) as f:
  96         for line in f:
  97             if line[0] == '#' or line == '\n':
  98                 continue
  99             defconfig = line.split()[6] + '_defconfig'
 100             if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
 101                 return False
 102
 103     return True
 104
 105 ### classes ###
 106 class KconfigScanner:
 107
 108     """Kconfig scanner."""
 109
 110     ### constant variable only used in this class ###
 111     _SYMBOL_TABLE = {
 112         'arch' : 'SYS_ARCH',
 113         'cpu' : 'SYS_CPU',
 114         'soc' : 'SYS_SOC',
 115         'vendor' : 'SYS_VENDOR',
 116         'board' : 'SYS_BOARD',
 117         'config' : 'SYS_CONFIG_NAME',
 118         'options' : 'SYS_EXTRA_OPTIONS'
 119     }
 120
 121     def __init__(self):
 122         """Scan all the Kconfig files and create a Config object."""
 123         # Define environment variables referenced from Kconfig
 124         os.environ['srctree'] = os.getcwd()
 125         os.environ['UBOOTVERSION'] = 'dummy'
 126         os.environ['KCONFIG_OBJDIR'] = ''
 127         self._conf = kconfiglib.Config()
 128
 129     def __del__(self):
 130         """Delete a leftover temporary file before exit.
 131
 132         The scan() method of this class creates a temporay file and deletes
 133         it on success.  If scan() method throws an exception on the way,
 134         the temporary file might be left over.  In that case, it should be
 135         deleted in this destructor.
 136         """
 137         if hasattr(self, '_tmpfile') and self._tmpfile:
 138             try_remove(self._tmpfile)
 139
 140     def scan(self, defconfig):
 141         """Load a defconfig file to obtain board parameters.
 142
 143         Arguments:
 144           defconfig: path to the defconfig file to be processed
 145
 146         Returns:
 147           A dictionary of board parameters.  It has a form of:
 148           {
 149               'arch': <arch_name>,
 150               'cpu': <cpu_name>,
 151               'soc': <soc_name>,
 152               'vendor': <vendor_name>,
 153               'board': <board_name>,
 154               'target': <target_name>,
 155               'config': <config_header_name>,
 156               'options': <extra_options>
 157           }
 158         """
 159         # strip special prefixes and save it in a temporary file
 160         fd, self._tmpfile = tempfile.mkstemp()
 161         with os.fdopen(fd, 'w') as f:
 162             for line in open(defconfig):
 163                 colon = line.find(':CONFIG_')
 164                 if colon == -1:
 165                     f.write(line)
 166                 else:
 167                     f.write(line[colon + 1:])
 168
 169         self._conf.load_config(self._tmpfile)
 170
 171         try_remove(self._tmpfile)
 172         self._tmpfile = None
 173
 174         params = {}
 175
 176         # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
 177         # Set '-' if the value is empty.
 178         for key, symbol in self._SYMBOL_TABLE.items():
 179             value = self._conf.get_symbol(symbol).get_value()
 180             if value:
 181                 params[key] = value
 182             else:
 183                 params[key] = '-'
 184
 185         defconfig = os.path.basename(defconfig)
 186         params['target'], match, rear = defconfig.partition('_defconfig')
 187         assert match and not rear, '%s : invalid defconfig' % defconfig
 188
 189         # fix-up for aarch64
 190         if params['arch'] == 'arm' and params['cpu'] == 'armv8':
 191             params['arch'] = 'aarch64'
 192
 193         # fix-up options field. It should have the form:
 194         # <config name>[:comma separated config options]
 195         if params['options'] != '-':
 196             params['options'] = params['config'] + ':' + \
 197                                 params['options'].replace(r'\"', '"')
 198         elif params['config'] != params['target']:
 199             params['options'] = params['config']
 200
 201         return params
 202
 203 def scan_defconfigs_for_multiprocess(queue, defconfigs):
 204     """Scan defconfig files and queue their board parameters
 205
 206     This function is intended to be passed to
 207     multiprocessing.Process() constructor.
 208
 209     Arguments:
 210       queue: An instance of multiprocessing.Queue().
 211              The resulting board parameters are written into it.
 212       defconfigs: A sequence of defconfig files to be scanned.
 213     """
 214     kconf_scanner = KconfigScanner()
 215     for defconfig in defconfigs:
 216         queue.put(kconf_scanner.scan(defconfig))
 217
 218 def read_queues(queues, params_list):
 219     """Read the queues and append the data to the paramers list"""
 220     for q in queues:
 221         while not q.empty():
 222             params_list.append(q.get())
 223
 224 def scan_defconfigs(jobs=1):
 225     """Collect board parameters for all defconfig files.
 226
 227     This function invokes multiple processes for faster processing.
 228
 229     Arguments:
 230       jobs: The number of jobs to run simultaneously
 231     """
 232     all_defconfigs = []
 233     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 234         for filename in fnmatch.filter(filenames, '*_defconfig'):
 235             if fnmatch.fnmatch(filename, '.*'):
 236                 continue
 237             all_defconfigs.append(os.path.join(dirpath, filename))
 238
 239     total_boards = len(all_defconfigs)
 240     processes = []
 241     queues = []
 242     for i in range(jobs):
 243         defconfigs = all_defconfigs[total_boards * i / jobs :
 244                                     total_boards * (i + 1) / jobs]
 245         q = multiprocessing.Queue(maxsize=-1)
 246         p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
 247                                     args=(q, defconfigs))
 248         p.start()
 249         processes.append(p)
 250         queues.append(q)
 251
 252     # The resulting data should be accumulated to this list
 253     params_list = []
 254
 255     # Data in the queues should be retrieved preriodically.
 256     # Otherwise, the queues would become full and subprocesses would get stuck.
 257     while any([p.is_alive() for p in processes]):
 258         read_queues(queues, params_list)
 259         # sleep for a while until the queues are filled
 260         time.sleep(SLEEP_TIME)
 261
 262     # Joining subprocesses just in case
 263     # (All subprocesses should already have been finished)
 264     for p in processes:
 265         p.join()
 266
 267     # retrieve leftover data
 268     read_queues(queues, params_list)
 269
 270     return params_list
 271
 272 class MaintainersDatabase:
 273
 274     """The database of board status and maintainers."""
 275
 276     def __init__(self):
 277         """Create an empty database."""
 278         self.database = {}
 279
 280     def get_status(self, target):
 281         """Return the status of the given board.
 282
 283         The board status is generally either 'Active' or 'Orphan'.
 284         Display a warning message and return '-' if status information
 285         is not found.
 286
 287         Returns:
 288           'Active', 'Orphan' or '-'.
 289         """
 290         if not target in self.database:
 291             print >> sys.stderr, "WARNING: no status info for '%s'" % target
 292             return '-'
 293
 294         tmp = self.database[target][0]
 295         if tmp.startswith('Maintained'):
 296             return 'Active'
 297         elif tmp.startswith('Orphan'):
 298             return 'Orphan'
 299         else:
 300             print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
 301                                   (tmp, target))
 302             return '-'
 303
 304     def get_maintainers(self, target):
 305         """Return the maintainers of the given board.
 306
 307         Returns:
 308           Maintainers of the board.  If the board has two or more maintainers,
 309           they are separated with colons.
 310         """
 311         if not target in self.database:
 312             print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
 313             return ''
 314
 315         return ':'.join(self.database[target][1])
 316
 317     def parse_file(self, file):
 318         """Parse a MAINTAINERS file.
 319
 320         Parse a MAINTAINERS file and accumulates board status and
 321         maintainers information.
 322
 323         Arguments:
 324           file: MAINTAINERS file to be parsed
 325         """
 326         targets = []
 327         maintainers = []
 328         status = '-'
 329         for line in open(file):
 330             # Check also commented maintainers
 331             if line[:3] == '#M:':
 332                 line = line[1:]
 333             tag, rest = line[:2], line[2:].strip()
 334             if tag == 'M:':
 335                 maintainers.append(rest)
 336             elif tag == 'F:':
 337                 # expand wildcard and filter by 'configs/*_defconfig'
 338                 for f in glob.glob(rest):
 339                     front, match, rear = f.partition('configs/')
 340                     if not front and match:
 341                         front, match, rear = rear.rpartition('_defconfig')
 342                         if match and not rear:
 343                             targets.append(front)
 344             elif tag == 'S:':
 345                 status = rest
 346             elif line == '\n':
 347                 for target in targets:
 348                     self.database[target] = (status, maintainers)
 349                 targets = []
 350                 maintainers = []
 351                 status = '-'
 352         if targets:
 353             for target in targets:
 354                 self.database[target] = (status, maintainers)
 355
 356 def insert_maintainers_info(params_list):
 357     """Add Status and Maintainers information to the board parameters list.
 358
 359     Arguments:
 360       params_list: A list of the board parameters
 361     """
 362     database = MaintainersDatabase()
 363     for (dirpath, dirnames, filenames) in os.walk('.'):
 364         if 'MAINTAINERS' in filenames:
 365             database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
 366
 367     for i, params in enumerate(params_list):
 368         target = params['target']
 369         params['status'] = database.get_status(target)
 370         params['maintainers'] = database.get_maintainers(target)
 371         params_list[i] = params
 372
 373 def format_and_output(params_list, output):
 374     """Write board parameters into a file.
 375
 376     Columnate the board parameters, sort lines alphabetically,
 377     and then write them to a file.
 378
 379     Arguments:
 380       params_list: The list of board parameters
 381       output: The path to the output file
 382     """
 383     FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
 384               'options', 'maintainers')
 385
 386     # First, decide the width of each column
 387     max_length = dict([ (f, 0) for f in FIELDS])
 388     for params in params_list:
 389         for f in FIELDS:
 390             max_length[f] = max(max_length[f], len(params[f]))
 391
 392     output_lines = []
 393     for params in params_list:
 394         line = ''
 395         for f in FIELDS:
 396             # insert two spaces between fields like column -t would
 397             line += '  ' + params[f].ljust(max_length[f])
 398         output_lines.append(line.strip())
 399
 400     # ignore case when sorting
 401     output_lines.sort(key=str.lower)
 402
 403     with open(output, 'w') as f:
 404         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 405
 406 def gen_boards_cfg(output, jobs=1, force=False):
 407     """Generate a board database file.
 408
 409     Arguments:
 410       output: The name of the output file
 411       jobs: The number of jobs to run simultaneously
 412       force: Force to generate the output even if it is new
 413     """
 414     check_top_directory()
 415
 416     if not force and output_is_new(output):
 417         print "%s is up to date. Nothing to do." % output
 418         sys.exit(0)
 419
 420     params_list = scan_defconfigs(jobs)
 421     insert_maintainers_info(params_list)
 422     format_and_output(params_list, output)
 423
 424 def main():
 425     try:
 426         cpu_count = multiprocessing.cpu_count()
 427     except NotImplementedError:
 428         cpu_count = 1
 429
 430     parser = optparse.OptionParser()
 431     # Add options here
 432     parser.add_option('-f', '--force', action="store_true", default=False,
 433                       help='regenerate the output even if it is new')
 434     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 435                       help='the number of jobs to run simultaneously')
 436     parser.add_option('-o', '--output', default=OUTPUT_FILE,
 437                       help='output file [default=%s]' % OUTPUT_FILE)
 438     (options, args) = parser.parse_args()
 439
 440     gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
 441
 442 if __name__ == '__main__':
 443     main()