tools/genboardscfg.py

   1 #!/usr/bin/env python
   2 #
   3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Converter from Kconfig and MAINTAINERS to boards.cfg
  10
  11 Run 'tools/genboardscfg.py' to create boards.cfg file.
  12
  13 Run 'tools/genboardscfg.py -h' for available options.
  14 """
  15
  16 import errno
  17 import fnmatch
  18 import glob
  19 import optparse
  20 import os
  21 import re
  22 import shutil
  23 import subprocess
  24 import sys
  25 import tempfile
  26 import time
  27
  28 BOARD_FILE = 'boards.cfg'
  29 CONFIG_DIR = 'configs'
  30 REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
  31                 '-i', '-d', '-', '-s', '8']
  32 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
  33 SLEEP_TIME=0.03
  34
  35 COMMENT_BLOCK = '''#
  36 # List of boards
  37 #   Automatically generated by %s: don't edit
  38 #
  39 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
  40
  41 ''' % __file__
  42
  43 ### helper functions ###
  44 def get_terminal_columns():
  45     """Get the width of the terminal.
  46
  47     Returns:
  48       The width of the terminal, or zero if the stdout is not
  49       associated with tty.
  50     """
  51     try:
  52         return shutil.get_terminal_size().columns # Python 3.3~
  53     except AttributeError:
  54         import fcntl
  55         import termios
  56         import struct
  57         arg = struct.pack('hhhh', 0, 0, 0, 0)
  58         try:
  59             ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
  60         except IOError as exception:
  61             # If 'Inappropriate ioctl for device' error occurs,
  62             # stdout is probably redirected. Return 0.
  63             return 0
  64         return struct.unpack('hhhh', ret)[1]
  65
  66 def get_devnull():
  67     """Get the file object of '/dev/null' device."""
  68     try:
  69         devnull = subprocess.DEVNULL # py3k
  70     except AttributeError:
  71         devnull = open(os.devnull, 'wb')
  72     return devnull
  73
  74 def check_top_directory():
  75     """Exit if we are not at the top of source directory."""
  76     for f in ('README', 'Licenses'):
  77         if not os.path.exists(f):
  78             sys.exit('Please run at the top of source directory.')
  79
  80 def get_make_cmd():
  81     """Get the command name of GNU Make."""
  82     process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
  83     ret = process.communicate()
  84     if process.returncode:
  85         sys.exit('GNU Make not found')
  86     return ret[0].rstrip()
  87
  88 ### classes ###
  89 class MaintainersDatabase:
  90
  91     """The database of board status and maintainers."""
  92
  93     def __init__(self):
  94         """Create an empty database."""
  95         self.database = {}
  96
  97     def get_status(self, target):
  98         """Return the status of the given board.
  99
 100         Returns:
 101           Either 'Active' or 'Orphan'
 102         """
 103         tmp = self.database[target][0]
 104         if tmp.startswith('Maintained'):
 105             return 'Active'
 106         elif tmp.startswith('Orphan'):
 107             return 'Orphan'
 108         else:
 109             print >> sys.stderr, 'Error: %s: unknown status' % tmp
 110
 111     def get_maintainers(self, target):
 112         """Return the maintainers of the given board.
 113
 114         If the board has two or more maintainers, they are separated
 115         with colons.
 116         """
 117         return ':'.join(self.database[target][1])
 118
 119     def parse_file(self, file):
 120         """Parse the given MAINTAINERS file.
 121
 122         This method parses MAINTAINERS and add board status and
 123         maintainers information to the database.
 124
 125         Arguments:
 126           file: MAINTAINERS file to be parsed
 127         """
 128         targets = []
 129         maintainers = []
 130         status = '-'
 131         for line in open(file):
 132             tag, rest = line[:2], line[2:].strip()
 133             if tag == 'M:':
 134                 maintainers.append(rest)
 135             elif tag == 'F:':
 136                 # expand wildcard and filter by 'configs/*_defconfig'
 137                 for f in glob.glob(rest):
 138                     front, match, rear = f.partition('configs/')
 139                     if not front and match:
 140                         front, match, rear = rear.rpartition('_defconfig')
 141                         if match and not rear:
 142                             targets.append(front)
 143             elif tag == 'S:':
 144                 status = rest
 145             elif line == '\n':
 146                 for target in targets:
 147                     self.database[target] = (status, maintainers)
 148                 targets = []
 149                 maintainers = []
 150                 status = '-'
 151         if targets:
 152             for target in targets:
 153                 self.database[target] = (status, maintainers)
 154
 155 class DotConfigParser:
 156
 157     """A parser of .config file.
 158
 159     Each line of the output should have the form of:
 160     Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
 161     Most of them are extracted from .config file.
 162     MAINTAINERS files are also consulted for Status and Maintainers fields.
 163     """
 164
 165     re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
 166     re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
 167     re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
 168     re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
 169     re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
 170     re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
 171     re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
 172     re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
 173                ('vendor', re_vendor), ('board', re_board),
 174                ('config', re_config), ('options', re_options))
 175     must_fields = ('arch', 'config')
 176
 177     def __init__(self, build_dir, output, maintainers_database):
 178         """Create a new .config perser.
 179
 180         Arguments:
 181           build_dir: Build directory where .config is located
 182           output: File object which the result is written to
 183           maintainers_database: An instance of class MaintainersDatabase
 184         """
 185         self.dotconfig = os.path.join(build_dir, '.config')
 186         self.output = output
 187         self.database = maintainers_database
 188
 189     def parse(self, defconfig):
 190         """Parse .config file and output one-line database for the given board.
 191
 192         Arguments:
 193           defconfig: Board (defconfig) name
 194         """
 195         fields = {}
 196         for line in open(self.dotconfig):
 197             if not line.startswith('CONFIG_SYS_'):
 198                 continue
 199             for (key, pattern) in self.re_list:
 200                 m = pattern.match(line)
 201                 if m and m.group(1):
 202                     fields[key] = m.group(1)
 203                     break
 204
 205         # sanity check of '.config' file
 206         for field in self.must_fields:
 207             if not field in fields:
 208                 sys.exit('Error: %s is not defined in %s' % (field, defconfig))
 209
 210         # fix-up for aarch64
 211         if fields['arch'] == 'arm' and 'cpu' in fields:
 212             if fields['cpu'] == 'armv8':
 213                 fields['arch'] = 'aarch64'
 214
 215         target, match, rear = defconfig.partition('_defconfig')
 216         assert match and not rear, \
 217                                 '%s : invalid defconfig file name' % defconfig
 218
 219         fields['status'] = self.database.get_status(target)
 220         fields['maintainers'] = self.database.get_maintainers(target)
 221
 222         if 'options' in fields:
 223             options = fields['config'] + ':' + \
 224                       fields['options'].replace(r'\"', '"')
 225         elif fields['config'] != target:
 226             options = fields['config']
 227         else:
 228             options = '-'
 229
 230         self.output.write((' '.join(['%s'] * 9) + '\n')  %
 231                           (fields['status'],
 232                            fields['arch'],
 233                            fields.get('cpu', '-'),
 234                            fields.get('soc', '-'),
 235                            fields.get('vendor', '-'),
 236                            fields.get('board', '-'),
 237                            target,
 238                            options,
 239                            fields['maintainers']))
 240
 241 class Slot:
 242
 243     """A slot to store a subprocess.
 244
 245     Each instance of this class handles one subprocess.
 246     This class is useful to control multiple processes
 247     for faster processing.
 248     """
 249
 250     def __init__(self, output, maintainers_database, devnull, make_cmd):
 251         """Create a new slot.
 252
 253         Arguments:
 254           output: File object which the result is written to
 255           maintainers_database: An instance of class MaintainersDatabase
 256         """
 257         self.occupied = False
 258         self.build_dir = tempfile.mkdtemp()
 259         self.devnull = devnull
 260         self.make_cmd = make_cmd
 261         self.parser = DotConfigParser(self.build_dir, output,
 262                                       maintainers_database)
 263
 264     def __del__(self):
 265         """Delete the working directory"""
 266         shutil.rmtree(self.build_dir)
 267
 268     def add(self, defconfig):
 269         """Add a new subprocess to the slot.
 270
 271         Fails if the slot is occupied, that is, the current subprocess
 272         is still running.
 273
 274         Arguments:
 275           defconfig: Board (defconfig) name
 276
 277         Returns:
 278           Return True on success or False on fail
 279         """
 280         if self.occupied:
 281             return False
 282         o = 'O=' + self.build_dir
 283         self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
 284                                    stdout=self.devnull)
 285         self.defconfig = defconfig
 286         self.occupied = True
 287         return True
 288
 289     def poll(self):
 290         """Check if the subprocess is running and invoke the .config
 291         parser if the subprocess is terminated.
 292
 293         Returns:
 294           Return True if the subprocess is terminated, False otherwise
 295         """
 296         if not self.occupied:
 297             return True
 298         if self.ps.poll() == None:
 299             return False
 300         self.parser.parse(self.defconfig)
 301         self.occupied = False
 302         return True
 303
 304 class Slots:
 305
 306     """Controller of the array of subprocess slots."""
 307
 308     def __init__(self, jobs, output, maintainers_database):
 309         """Create a new slots controller.
 310
 311         Arguments:
 312           jobs: A number of slots to instantiate
 313           output: File object which the result is written to
 314           maintainers_database: An instance of class MaintainersDatabase
 315         """
 316         self.slots = []
 317         devnull = get_devnull()
 318         make_cmd = get_make_cmd()
 319         for i in range(jobs):
 320             self.slots.append(Slot(output, maintainers_database,
 321                                    devnull, make_cmd))
 322
 323     def add(self, defconfig):
 324         """Add a new subprocess if a vacant slot is available.
 325
 326         Arguments:
 327           defconfig: Board (defconfig) name
 328
 329         Returns:
 330           Return True on success or False on fail
 331         """
 332         for slot in self.slots:
 333             if slot.add(defconfig):
 334                 return True
 335         return False
 336
 337     def available(self):
 338         """Check if there is a vacant slot.
 339
 340         Returns:
 341           Return True if a vacant slot is found, False if all slots are full
 342         """
 343         for slot in self.slots:
 344             if slot.poll():
 345                 return True
 346         return False
 347
 348     def empty(self):
 349         """Check if all slots are vacant.
 350
 351         Returns:
 352           Return True if all slots are vacant, False if at least one slot
 353           is running
 354         """
 355         ret = True
 356         for slot in self.slots:
 357             if not slot.poll():
 358                 ret = False
 359         return ret
 360
 361 class Indicator:
 362
 363     """A class to control the progress indicator."""
 364
 365     MIN_WIDTH = 15
 366     MAX_WIDTH = 70
 367
 368     def __init__(self, total):
 369         """Create an instance.
 370
 371         Arguments:
 372           total: A number of boards
 373         """
 374         self.total = total
 375         self.cur = 0
 376         width = get_terminal_columns()
 377         width = min(width, self.MAX_WIDTH)
 378         width -= self.MIN_WIDTH
 379         if width > 0:
 380             self.enabled = True
 381         else:
 382             self.enabled = False
 383         self.width = width
 384
 385     def inc(self):
 386         """Increment the counter and show the progress bar."""
 387         if not self.enabled:
 388             return
 389         self.cur += 1
 390         arrow_len = self.width * self.cur // self.total
 391         msg = '%4d/%d [' % (self.cur, self.total)
 392         msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
 393         sys.stdout.write('\r' + msg)
 394         sys.stdout.flush()
 395
 396 def __gen_boards_cfg(jobs):
 397     """Generate boards.cfg file.
 398
 399     Arguments:
 400       jobs: The number of jobs to run simultaneously
 401
 402     Note:
 403       The incomplete boards.cfg is left over when an error (including
 404       the termination by the keyboard interrupt) occurs on the halfway.
 405     """
 406     check_top_directory()
 407     print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)
 408
 409     # All the defconfig files to be processed
 410     defconfigs = []
 411     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 412         dirpath = dirpath[len(CONFIG_DIR) + 1:]
 413         for filename in fnmatch.filter(filenames, '*_defconfig'):
 414             if fnmatch.fnmatch(filename, '.*'):
 415                 continue
 416             defconfigs.append(os.path.join(dirpath, filename))
 417
 418     # Parse all the MAINTAINERS files
 419     maintainers_database = MaintainersDatabase()
 420     for (dirpath, dirnames, filenames) in os.walk('.'):
 421         if 'MAINTAINERS' in filenames:
 422             maintainers_database.parse_file(os.path.join(dirpath,
 423                                                          'MAINTAINERS'))
 424
 425     # Output lines should be piped into the reformat tool
 426     reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
 427                                         stdout=open(BOARD_FILE, 'w'))
 428     pipe = reformat_process.stdin
 429     pipe.write(COMMENT_BLOCK)
 430
 431     indicator = Indicator(len(defconfigs))
 432     slots = Slots(jobs, pipe, maintainers_database)
 433
 434     # Main loop to process defconfig files:
 435     #  Add a new subprocess into a vacant slot.
 436     #  Sleep if there is no available slot.
 437     for defconfig in defconfigs:
 438         while not slots.add(defconfig):
 439             while not slots.available():
 440                 # No available slot: sleep for a while
 441                 time.sleep(SLEEP_TIME)
 442         indicator.inc()
 443
 444     # wait until all the subprocesses finish
 445     while not slots.empty():
 446         time.sleep(SLEEP_TIME)
 447     print ''
 448
 449     # wait until the reformat tool finishes
 450     reformat_process.communicate()
 451     if reformat_process.returncode != 0:
 452         sys.exit('"%s" failed' % REFORMAT_CMD[0])
 453
 454 def gen_boards_cfg(jobs):
 455     """Generate boards.cfg file.
 456
 457     The incomplete boards.cfg is deleted if an error (including
 458     the termination by the keyboard interrupt) occurs on the halfway.
 459
 460     Arguments:
 461       jobs: The number of jobs to run simultaneously
 462     """
 463     try:
 464         __gen_boards_cfg(jobs)
 465     except:
 466         # We should remove incomplete boards.cfg
 467         try:
 468             os.remove(BOARD_FILE)
 469         except OSError as exception:
 470             # Ignore 'No such file or directory' error
 471             if exception.errno != errno.ENOENT:
 472                 raise
 473         raise
 474
 475 def main():
 476     parser = optparse.OptionParser()
 477     # Add options here
 478     parser.add_option('-j', '--jobs',
 479                       help='the number of jobs to run simultaneously')
 480     (options, args) = parser.parse_args()
 481     if options.jobs:
 482         try:
 483             jobs = int(options.jobs)
 484         except ValueError:
 485             sys.exit('Option -j (--jobs) takes a number')
 486     else:
 487         try:
 488             jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
 489                                      stdout=subprocess.PIPE).communicate()[0])
 490         except (OSError, ValueError):
 491             print 'info: failed to get the number of CPUs. Set jobs to 1'
 492             jobs = 1
 493     gen_boards_cfg(jobs)
 494
 495 if __name__ == '__main__':
 496     main()