tools/genboardscfg.py

   1 #!/usr/bin/env python
   2 #
   3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Converter from Kconfig and MAINTAINERS to boards.cfg
  10
  11 Run 'tools/genboardscfg.py' to create boards.cfg file.
  12
  13 Run 'tools/genboardscfg.py -h' for available options.
  14 """
  15
  16 import errno
  17 import fnmatch
  18 import glob
  19 import optparse
  20 import os
  21 import re
  22 import shutil
  23 import subprocess
  24 import sys
  25 import tempfile
  26 import time
  27
  28 BOARD_FILE = 'boards.cfg'
  29 CONFIG_DIR = 'configs'
  30 REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
  31                 '-i', '-d', '-', '-s', '8']
  32 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
  33 SLEEP_TIME=0.03
  34
  35 COMMENT_BLOCK = '''#
  36 # List of boards
  37 #   Automatically generated by %s: don't edit
  38 #
  39 # Status, Arch, CPU(:SPLCPU), SoC, Vendor, Board, Target, Options, Maintainers
  40
  41 ''' % __file__
  42
  43 ### helper functions ###
  44 def get_terminal_columns():
  45     """Get the width of the terminal.
  46
  47     Returns:
  48       The width of the terminal, or zero if the stdout is not
  49       associated with tty.
  50     """
  51     try:
  52         return shutil.get_terminal_size().columns # Python 3.3~
  53     except AttributeError:
  54         import fcntl
  55         import termios
  56         import struct
  57         arg = struct.pack('hhhh', 0, 0, 0, 0)
  58         try:
  59             ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
  60         except IOError as exception:
  61             if exception.errno != errno.ENOTTY:
  62                 raise
  63             # If 'Inappropriate ioctl for device' error occurs,
  64             # stdout is probably redirected. Return 0.
  65             return 0
  66         return struct.unpack('hhhh', ret)[1]
  67
  68 def get_devnull():
  69     """Get the file object of '/dev/null' device."""
  70     try:
  71         devnull = subprocess.DEVNULL # py3k
  72     except AttributeError:
  73         devnull = open(os.devnull, 'wb')
  74     return devnull
  75
  76 def check_top_directory():
  77     """Exit if we are not at the top of source directory."""
  78     for f in ('README', 'Licenses'):
  79         if not os.path.exists(f):
  80             sys.exit('Please run at the top of source directory.')
  81
  82 def get_make_cmd():
  83     """Get the command name of GNU Make."""
  84     process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
  85     ret = process.communicate()
  86     if process.returncode:
  87         sys.exit('GNU Make not found')
  88     return ret[0].rstrip()
  89
  90 ### classes ###
  91 class MaintainersDatabase:
  92
  93     """The database of board status and maintainers."""
  94
  95     def __init__(self):
  96         """Create an empty database."""
  97         self.database = {}
  98
  99     def get_status(self, target):
 100         """Return the status of the given board.
 101
 102         Returns:
 103           Either 'Active' or 'Orphan'
 104         """
 105         tmp = self.database[target][0]
 106         if tmp.startswith('Maintained'):
 107             return 'Active'
 108         elif tmp.startswith('Orphan'):
 109             return 'Orphan'
 110         else:
 111             print >> sys.stderr, 'Error: %s: unknown status' % tmp
 112
 113     def get_maintainers(self, target):
 114         """Return the maintainers of the given board.
 115
 116         If the board has two or more maintainers, they are separated
 117         with colons.
 118         """
 119         return ':'.join(self.database[target][1])
 120
 121     def parse_file(self, file):
 122         """Parse the given MAINTAINERS file.
 123
 124         This method parses MAINTAINERS and add board status and
 125         maintainers information to the database.
 126
 127         Arguments:
 128           file: MAINTAINERS file to be parsed
 129         """
 130         targets = []
 131         maintainers = []
 132         status = '-'
 133         for line in open(file):
 134             tag, rest = line[:2], line[2:].strip()
 135             if tag == 'M:':
 136                 maintainers.append(rest)
 137             elif tag == 'F:':
 138                 # expand wildcard and filter by 'configs/*_defconfig'
 139                 for f in glob.glob(rest):
 140                     front, match, rear = f.partition('configs/')
 141                     if not front and match:
 142                         front, match, rear = rear.rpartition('_defconfig')
 143                         if match and not rear:
 144                             targets.append(front)
 145             elif tag == 'S:':
 146                 status = rest
 147             elif line == '\n' and targets:
 148                 for target in targets:
 149                     self.database[target] = (status, maintainers)
 150                 targets = []
 151                 maintainers = []
 152                 status = '-'
 153         if targets:
 154             for target in targets:
 155                 self.database[target] = (status, maintainers)
 156
 157 class DotConfigParser:
 158
 159     """A parser of .config file.
 160
 161     Each line of the output should have the form of:
 162     Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
 163     Most of them are extracted from .config file.
 164     MAINTAINERS files are also consulted for Status and Maintainers fields.
 165     """
 166
 167     re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
 168     re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
 169     re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
 170     re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
 171     re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
 172     re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
 173     re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
 174     re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
 175                ('vendor', re_vendor), ('board', re_board),
 176                ('config', re_config), ('options', re_options))
 177     must_fields = ('arch', 'config')
 178
 179     def __init__(self, build_dir, output, maintainers_database):
 180         """Create a new .config perser.
 181
 182         Arguments:
 183           build_dir: Build directory where .config is located
 184           output: File object which the result is written to
 185           maintainers_database: An instance of class MaintainersDatabase
 186         """
 187         self.dotconfig = os.path.join(build_dir, '.config')
 188         self.output = output
 189         self.database = maintainers_database
 190
 191     def parse(self, defconfig):
 192         """Parse .config file and output one-line database for the given board.
 193
 194         Arguments:
 195           defconfig: Board (defconfig) name
 196         """
 197         fields = {}
 198         for line in open(self.dotconfig):
 199             if not line.startswith('CONFIG_SYS_'):
 200                 continue
 201             for (key, pattern) in self.re_list:
 202                 m = pattern.match(line)
 203                 if m and m.group(1):
 204                     fields[key] = m.group(1)
 205                     break
 206
 207         # sanity check of '.config' file
 208         for field in self.must_fields:
 209             if not field in fields:
 210                 sys.exit('Error: %s is not defined in %s' % (field, defconfig))
 211
 212         # fix-up for aarch64 and tegra
 213         if fields['arch'] == 'arm' and 'cpu' in fields:
 214             if fields['cpu'] == 'armv8':
 215                 fields['arch'] = 'aarch64'
 216             if 'soc' in fields and re.match('tegra[0-9]*$', fields['soc']):
 217                 fields['cpu'] += ':arm720t'
 218
 219         target, match, rear = defconfig.partition('_defconfig')
 220         assert match and not rear, \
 221                                 '%s : invalid defconfig file name' % defconfig
 222
 223         fields['status'] = self.database.get_status(target)
 224         fields['maintainers'] = self.database.get_maintainers(target)
 225
 226         if 'options' in fields:
 227             options = fields['config'] + ':' + \
 228                       fields['options'].replace(r'\"', '"')
 229         elif fields['config'] != target:
 230             options = fields['config']
 231         else:
 232             options = '-'
 233
 234         self.output.write((' '.join(['%s'] * 9) + '\n')  %
 235                           (fields['status'],
 236                            fields['arch'],
 237                            fields.get('cpu', '-'),
 238                            fields.get('soc', '-'),
 239                            fields.get('vendor', '-'),
 240                            fields.get('board', '-'),
 241                            target,
 242                            options,
 243                            fields['maintainers']))
 244
 245 class Slot:
 246
 247     """A slot to store a subprocess.
 248
 249     Each instance of this class handles one subprocess.
 250     This class is useful to control multiple processes
 251     for faster processing.
 252     """
 253
 254     def __init__(self, output, maintainers_database, devnull, make_cmd):
 255         """Create a new slot.
 256
 257         Arguments:
 258           output: File object which the result is written to
 259           maintainers_database: An instance of class MaintainersDatabase
 260         """
 261         self.occupied = False
 262         self.build_dir = tempfile.mkdtemp()
 263         self.devnull = devnull
 264         self.make_cmd = make_cmd
 265         self.parser = DotConfigParser(self.build_dir, output,
 266                                       maintainers_database)
 267
 268     def __del__(self):
 269         """Delete the working directory"""
 270         shutil.rmtree(self.build_dir)
 271
 272     def add(self, defconfig):
 273         """Add a new subprocess to the slot.
 274
 275         Fails if the slot is occupied, that is, the current subprocess
 276         is still running.
 277
 278         Arguments:
 279           defconfig: Board (defconfig) name
 280
 281         Returns:
 282           Return True on success or False on fail
 283         """
 284         if self.occupied:
 285             return False
 286         o = 'O=' + self.build_dir
 287         self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
 288                                    stdout=self.devnull)
 289         self.defconfig = defconfig
 290         self.occupied = True
 291         return True
 292
 293     def poll(self):
 294         """Check if the subprocess is running and invoke the .config
 295         parser if the subprocess is terminated.
 296
 297         Returns:
 298           Return True if the subprocess is terminated, False otherwise
 299         """
 300         if not self.occupied:
 301             return True
 302         if self.ps.poll() == None:
 303             return False
 304         self.parser.parse(self.defconfig)
 305         self.occupied = False
 306         return True
 307
 308 class Slots:
 309
 310     """Controller of the array of subprocess slots."""
 311
 312     def __init__(self, jobs, output, maintainers_database):
 313         """Create a new slots controller.
 314
 315         Arguments:
 316           jobs: A number of slots to instantiate
 317           output: File object which the result is written to
 318           maintainers_database: An instance of class MaintainersDatabase
 319         """
 320         self.slots = []
 321         devnull = get_devnull()
 322         make_cmd = get_make_cmd()
 323         for i in range(jobs):
 324             self.slots.append(Slot(output, maintainers_database,
 325                                    devnull, make_cmd))
 326
 327     def add(self, defconfig):
 328         """Add a new subprocess if a vacant slot is available.
 329
 330         Arguments:
 331           defconfig: Board (defconfig) name
 332
 333         Returns:
 334           Return True on success or False on fail
 335         """
 336         for slot in self.slots:
 337             if slot.add(defconfig):
 338                 return True
 339         return False
 340
 341     def available(self):
 342         """Check if there is a vacant slot.
 343
 344         Returns:
 345           Return True if a vacant slot is found, False if all slots are full
 346         """
 347         for slot in self.slots:
 348             if slot.poll():
 349                 return True
 350         return False
 351
 352     def empty(self):
 353         """Check if all slots are vacant.
 354
 355         Returns:
 356           Return True if all slots are vacant, False if at least one slot
 357           is running
 358         """
 359         ret = True
 360         for slot in self.slots:
 361             if not slot.poll():
 362                 ret = False
 363         return ret
 364
 365 class Indicator:
 366
 367     """A class to control the progress indicator."""
 368
 369     MIN_WIDTH = 15
 370     MAX_WIDTH = 70
 371
 372     def __init__(self, total):
 373         """Create an instance.
 374
 375         Arguments:
 376           total: A number of boards
 377         """
 378         self.total = total
 379         self.cur = 0
 380         width = get_terminal_columns()
 381         width = min(width, self.MAX_WIDTH)
 382         width -= self.MIN_WIDTH
 383         if width > 0:
 384             self.enabled = True
 385         else:
 386             self.enabled = False
 387         self.width = width
 388
 389     def inc(self):
 390         """Increment the counter and show the progress bar."""
 391         if not self.enabled:
 392             return
 393         self.cur += 1
 394         arrow_len = self.width * self.cur // self.total
 395         msg = '%4d/%d [' % (self.cur, self.total)
 396         msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
 397         sys.stdout.write('\r' + msg)
 398         sys.stdout.flush()
 399
 400 def __gen_boards_cfg(jobs):
 401     """Generate boards.cfg file.
 402
 403     Arguments:
 404       jobs: The number of jobs to run simultaneously
 405
 406     Note:
 407       The incomplete boards.cfg is left over when an error (including
 408       the termination by the keyboard interrupt) occurs on the halfway.
 409     """
 410     check_top_directory()
 411     print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)
 412
 413     # All the defconfig files to be processed
 414     defconfigs = []
 415     for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
 416         dirpath = dirpath[len(CONFIG_DIR) + 1:]
 417         for filename in fnmatch.filter(filenames, '*_defconfig'):
 418             defconfigs.append(os.path.join(dirpath, filename))
 419
 420     # Parse all the MAINTAINERS files
 421     maintainers_database = MaintainersDatabase()
 422     for (dirpath, dirnames, filenames) in os.walk('.'):
 423         if 'MAINTAINERS' in filenames:
 424             maintainers_database.parse_file(os.path.join(dirpath,
 425                                                          'MAINTAINERS'))
 426
 427     # Output lines should be piped into the reformat tool
 428     reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
 429                                         stdout=open(BOARD_FILE, 'w'))
 430     pipe = reformat_process.stdin
 431     pipe.write(COMMENT_BLOCK)
 432
 433     indicator = Indicator(len(defconfigs))
 434     slots = Slots(jobs, pipe, maintainers_database)
 435
 436     # Main loop to process defconfig files:
 437     #  Add a new subprocess into a vacant slot.
 438     #  Sleep if there is no available slot.
 439     for defconfig in defconfigs:
 440         while not slots.add(defconfig):
 441             while not slots.available():
 442                 # No available slot: sleep for a while
 443                 time.sleep(SLEEP_TIME)
 444         indicator.inc()
 445
 446     # wait until all the subprocesses finish
 447     while not slots.empty():
 448         time.sleep(SLEEP_TIME)
 449     print ''
 450
 451     # wait until the reformat tool finishes
 452     reformat_process.communicate()
 453     if reformat_process.returncode != 0:
 454         sys.exit('"%s" failed' % REFORMAT_CMD[0])
 455
 456 def gen_boards_cfg(jobs):
 457     """Generate boards.cfg file.
 458
 459     The incomplete boards.cfg is deleted if an error (including
 460     the termination by the keyboard interrupt) occurs on the halfway.
 461
 462     Arguments:
 463       jobs: The number of jobs to run simultaneously
 464     """
 465     try:
 466         __gen_boards_cfg(jobs)
 467     except:
 468         # We should remove incomplete boards.cfg
 469         try:
 470             os.remove(BOARD_FILE)
 471         except OSError as exception:
 472             # Ignore 'No such file or directory' error
 473             if exception.errno != errno.ENOENT:
 474                 raise
 475         raise
 476
 477 def main():
 478     parser = optparse.OptionParser()
 479     # Add options here
 480     parser.add_option('-j', '--jobs',
 481                       help='the number of jobs to run simultaneously')
 482     (options, args) = parser.parse_args()
 483     if options.jobs:
 484         try:
 485             jobs = int(options.jobs)
 486         except ValueError:
 487             sys.exit('Option -j (--jobs) takes a number')
 488     else:
 489         try:
 490             jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
 491                                      stdout=subprocess.PIPE).communicate()[0])
 492         except (OSError, ValueError):
 493             print 'info: failed to get the number of CPUs. Set jobs to 1'
 494             jobs = 1
 495     gen_boards_cfg(jobs)
 496
 497 if __name__ == '__main__':
 498     main()