tools/moveconfig.py

   1 #!/usr/bin/env python2
   2 #
   3 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Move config options from headers to defconfig files.
  10
  11 Since Kconfig was introduced to U-Boot, we have worked on moving
  12 config options from headers to Kconfig (defconfig).
  13
  14 This tool intends to help this tremendous work.
  15
  16
  17 Usage
  18 -----
  19
  20 First, you must edit the Kconfig to add the menu entries for the configs
  21 you are moving.
  22
  23 And then run this tool giving CONFIG names you want to move.
  24 For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE,
  25 simply type as follows:
  26
  27   $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE
  28
  29 The tool walks through all the defconfig files and move the given CONFIGs.
  30
  31 The log is also displayed on the terminal.
  32
  33 Each line is printed in the format
  34 <defconfig_name>   :  <action>
  35
  36 <defconfig_name> is the name of the defconfig
  37 (without the suffix _defconfig).
  38
  39 <action> shows what the tool did for that defconfig.
  40 It looks like one of the followings:
  41
  42  - Move 'CONFIG_... '
  43    This config option was moved to the defconfig
  44
  45  - CONFIG_... is not defined in Kconfig.  Do nothing.
  46    The entry for this CONFIG was not found in Kconfig.
  47    There are two common cases:
  48      - You forgot to create an entry for the CONFIG before running
  49        this tool, or made a typo in a CONFIG passed to this tool.
  50      - The entry was hidden due to unmet 'depends on'.
  51        This is correct behavior.
  52
  53  - 'CONFIG_...' is the same as the define in Kconfig.  Do nothing.
  54    The define in the config header matched the one in Kconfig.
  55    We do not need to touch it.
  56
  57  - Undefined.  Do nothing.
  58    This config option was not found in the config header.
  59    Nothing to do.
  60
  61  - Compiler is missing.  Do nothing.
  62    The compiler specified for this architecture was not found
  63    in your PATH environment.
  64    (If -e option is passed, the tool exits immediately.)
  65
  66  - Failed to process.
  67    An error occurred during processing this defconfig.  Skipped.
  68    (If -e option is passed, the tool exits immediately on error.)
  69
  70 Finally, you will be asked, Clean up headers? [y/n]:
  71
  72 If you say 'y' here, the unnecessary config defines are removed
  73 from the config headers (include/configs/*.h).
  74 It just uses the regex method, so you should not rely on it.
  75 Just in case, please do 'git diff' to see what happened.
  76
  77
  78 How does it work?
  79 -----------------
  80
  81 This tool runs configuration and builds include/autoconf.mk for every
  82 defconfig.  The config options defined in Kconfig appear in the .config
  83 file (unless they are hidden because of unmet dependency.)
  84 On the other hand, the config options defined by board headers are seen
  85 in include/autoconf.mk.  The tool looks for the specified options in both
  86 of them to decide the appropriate action for the options.  If the given
  87 config option is found in the .config, but its value does not match the
  88 one from the board header, the config option in the .config is replaced
  89 with the define in the board header.  Then, the .config is synced by
  90 "make savedefconfig" and the defconfig is updated with it.
  91
  92 For faster processing, this tool handles multi-threading.  It creates
  93 separate build directories where the out-of-tree build is run.  The
  94 temporary build directories are automatically created and deleted as
  95 needed.  The number of threads are chosen based on the number of the CPU
  96 cores of your system although you can change it via -j (--jobs) option.
  97
  98
  99 Toolchains
 100 ----------
 101
 102 Appropriate toolchain are necessary to generate include/autoconf.mk
 103 for all the architectures supported by U-Boot.  Most of them are available
 104 at the kernel.org site, some are not provided by kernel.org.
 105
 106 The default per-arch CROSS_COMPILE used by this tool is specified by
 107 the list below, CROSS_COMPILE.  You may wish to update the list to
 108 use your own.  Instead of modifying the list directly, you can give
 109 them via environments.
 110
 111
 112 Available options
 113 -----------------
 114
 115  -c, --color
 116    Surround each portion of the log with escape sequences to display it
 117    in color on the terminal.
 118
 119  -d, --defconfigs
 120   Specify a file containing a list of defconfigs to move
 121
 122  -n, --dry-run
 123    Perform a trial run that does not make any changes.  It is useful to
 124    see what is going to happen before one actually runs it.
 125
 126  -e, --exit-on-error
 127    Exit immediately if Make exits with a non-zero status while processing
 128    a defconfig file.
 129
 130  -H, --headers-only
 131    Only cleanup the headers; skip the defconfig processing
 132
 133  -j, --jobs
 134    Specify the number of threads to run simultaneously.  If not specified,
 135    the number of threads is the same as the number of CPU cores.
 136
 137  -v, --verbose
 138    Show any build errors as boards are built
 139
 140 To see the complete list of supported options, run
 141
 142   $ tools/moveconfig.py -h
 143
 144 """
 145
 146 import fnmatch
 147 import multiprocessing
 148 import optparse
 149 import os
 150 import re
 151 import shutil
 152 import subprocess
 153 import sys
 154 import tempfile
 155 import time
 156
 157 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
 158 SLEEP_TIME=0.03
 159
 160 # Here is the list of cross-tools I use.
 161 # Most of them are available at kernel.org
 162 # (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
 163 # arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
 164 # blackfin: http://sourceforge.net/projects/adi-toolchain/files/
 165 # nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz
 166 # nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545
 167 # sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu
 168 #
 169 # openrisc kernel.org toolchain is out of date, download latest one from
 170 # http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions
 171 CROSS_COMPILE = {
 172     'arc': 'arc-linux-',
 173     'aarch64': 'aarch64-linux-',
 174     'arm': 'arm-unknown-linux-gnueabi-',
 175     'avr32': 'avr32-linux-',
 176     'blackfin': 'bfin-elf-',
 177     'm68k': 'm68k-linux-',
 178     'microblaze': 'microblaze-linux-',
 179     'mips': 'mips-linux-',
 180     'nds32': 'nds32le-linux-',
 181     'nios2': 'nios2-linux-gnu-',
 182     'openrisc': 'or1k-elf-',
 183     'powerpc': 'powerpc-linux-',
 184     'sh': 'sh-linux-gnu-',
 185     'sparc': 'sparc-linux-',
 186     'x86': 'i386-linux-'
 187 }
 188
 189 STATE_IDLE = 0
 190 STATE_DEFCONFIG = 1
 191 STATE_AUTOCONF = 2
 192 STATE_SAVEDEFCONFIG = 3
 193
 194 ACTION_MOVE = 0
 195 ACTION_NO_ENTRY = 1
 196 ACTION_NO_CHANGE = 2
 197
 198 COLOR_BLACK        = '0;30'
 199 COLOR_RED          = '0;31'
 200 COLOR_GREEN        = '0;32'
 201 COLOR_BROWN        = '0;33'
 202 COLOR_BLUE         = '0;34'
 203 COLOR_PURPLE       = '0;35'
 204 COLOR_CYAN         = '0;36'
 205 COLOR_LIGHT_GRAY   = '0;37'
 206 COLOR_DARK_GRAY    = '1;30'
 207 COLOR_LIGHT_RED    = '1;31'
 208 COLOR_LIGHT_GREEN  = '1;32'
 209 COLOR_YELLOW       = '1;33'
 210 COLOR_LIGHT_BLUE   = '1;34'
 211 COLOR_LIGHT_PURPLE = '1;35'
 212 COLOR_LIGHT_CYAN   = '1;36'
 213 COLOR_WHITE        = '1;37'
 214
 215 ### helper functions ###
 216 def get_devnull():
 217     """Get the file object of '/dev/null' device."""
 218     try:
 219         devnull = subprocess.DEVNULL # py3k
 220     except AttributeError:
 221         devnull = open(os.devnull, 'wb')
 222     return devnull
 223
 224 def check_top_directory():
 225     """Exit if we are not at the top of source directory."""
 226     for f in ('README', 'Licenses'):
 227         if not os.path.exists(f):
 228             sys.exit('Please run at the top of source directory.')
 229
 230 def check_clean_directory():
 231     """Exit if the source tree is not clean."""
 232     for f in ('.config', 'include/config'):
 233         if os.path.exists(f):
 234             sys.exit("source tree is not clean, please run 'make mrproper'")
 235
 236 def get_make_cmd():
 237     """Get the command name of GNU Make.
 238
 239     U-Boot needs GNU Make for building, but the command name is not
 240     necessarily "make". (for example, "gmake" on FreeBSD).
 241     Returns the most appropriate command name on your system.
 242     """
 243     process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
 244     ret = process.communicate()
 245     if process.returncode:
 246         sys.exit('GNU Make not found')
 247     return ret[0].rstrip()
 248
 249 def color_text(color_enabled, color, string):
 250     """Return colored string."""
 251     if color_enabled:
 252         return '\033[' + color + 'm' + string + '\033[0m'
 253     else:
 254         return string
 255
 256 def log_msg(color_enabled, color, defconfig, msg):
 257     """Return the formated line for the log."""
 258     return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \
 259         color_text(color_enabled, color, msg) + '\n'
 260
 261 def update_cross_compile(color_enabled):
 262     """Update per-arch CROSS_COMPILE via environment variables
 263
 264     The default CROSS_COMPILE values are available
 265     in the CROSS_COMPILE list above.
 266
 267     You can override them via environment variables
 268     CROSS_COMPILE_{ARCH}.
 269
 270     For example, if you want to override toolchain prefixes
 271     for ARM and PowerPC, you can do as follows in your shell:
 272
 273     export CROSS_COMPILE_ARM=...
 274     export CROSS_COMPILE_POWERPC=...
 275
 276     Then, this function checks if specified compilers really exist in your
 277     PATH environment.
 278     """
 279     archs = []
 280
 281     for arch in os.listdir('arch'):
 282         if os.path.exists(os.path.join('arch', arch, 'Makefile')):
 283             archs.append(arch)
 284
 285     # arm64 is a special case
 286     archs.append('aarch64')
 287
 288     for arch in archs:
 289         env = 'CROSS_COMPILE_' + arch.upper()
 290         cross_compile = os.environ.get(env)
 291         if not cross_compile:
 292             cross_compile = CROSS_COMPILE.get(arch, '')
 293
 294         for path in os.environ["PATH"].split(os.pathsep):
 295             gcc_path = os.path.join(path, cross_compile + 'gcc')
 296             if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK):
 297                 break
 298         else:
 299             print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW,
 300                  'warning: %sgcc: not found in PATH.  %s architecture boards will be skipped'
 301                                             % (cross_compile, arch))
 302             cross_compile = None
 303
 304         CROSS_COMPILE[arch] = cross_compile
 305
 306 def cleanup_one_header(header_path, patterns, dry_run):
 307     """Clean regex-matched lines away from a file.
 308
 309     Arguments:
 310       header_path: path to the cleaned file.
 311       patterns: list of regex patterns.  Any lines matching to these
 312                 patterns are deleted.
 313       dry_run: make no changes, but still display log.
 314     """
 315     with open(header_path) as f:
 316         lines = f.readlines()
 317
 318     matched = []
 319     for i, line in enumerate(lines):
 320         for pattern in patterns:
 321             m = pattern.search(line)
 322             if m:
 323                 print '%s: %s: %s' % (header_path, i + 1, line),
 324                 matched.append(i)
 325                 break
 326
 327     if dry_run or not matched:
 328         return
 329
 330     with open(header_path, 'w') as f:
 331         for i, line in enumerate(lines):
 332             if not i in matched:
 333                 f.write(line)
 334
 335 def cleanup_headers(configs, dry_run):
 336     """Delete config defines from board headers.
 337
 338     Arguments:
 339       configs: A list of CONFIGs to remove.
 340       dry_run: make no changes, but still display log.
 341     """
 342     while True:
 343         choice = raw_input('Clean up headers? [y/n]: ').lower()
 344         print choice
 345         if choice == 'y' or choice == 'n':
 346             break
 347
 348     if choice == 'n':
 349         return
 350
 351     patterns = []
 352     for config in configs:
 353         patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
 354         patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
 355
 356     for dir in 'include', 'arch', 'board':
 357         for (dirpath, dirnames, filenames) in os.walk(dir):
 358             for filename in filenames:
 359                 if not fnmatch.fnmatch(filename, '*~'):
 360                     cleanup_one_header(os.path.join(dirpath, filename),
 361                                        patterns, dry_run)
 362
 363 ### classes ###
 364 class Progress:
 365
 366     """Progress Indicator"""
 367
 368     def __init__(self, total):
 369         """Create a new progress indicator.
 370
 371         Arguments:
 372           total: A number of defconfig files to process.
 373         """
 374         self.current = 0
 375         self.total = total
 376
 377     def inc(self):
 378         """Increment the number of processed defconfig files."""
 379
 380         self.current += 1
 381
 382     def show(self):
 383         """Display the progress."""
 384         print ' %d defconfigs out of %d\r' % (self.current, self.total),
 385         sys.stdout.flush()
 386
 387 class KconfigParser:
 388
 389     """A parser of .config and include/autoconf.mk."""
 390
 391     re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
 392     re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
 393
 394     def __init__(self, configs, options, progress, build_dir):
 395         """Create a new parser.
 396
 397         Arguments:
 398           configs: A list of CONFIGs to move.
 399           options: option flags.
 400           progress: A progress indicator
 401           build_dir: Build directory.
 402         """
 403         self.configs = configs
 404         self.options = options
 405         self.progress = progress
 406         self.build_dir = build_dir
 407
 408     def get_cross_compile(self):
 409         """Parse .config file and return CROSS_COMPILE.
 410
 411         Returns:
 412           A string storing the compiler prefix for the architecture.
 413           Return a NULL string for architectures that do not require
 414           compiler prefix (Sandbox and native build is the case).
 415           Return None if the specified compiler is missing in your PATH.
 416           Caller should distinguish '' and None.
 417         """
 418         arch = ''
 419         cpu = ''
 420         dotconfig = os.path.join(self.build_dir, '.config')
 421         for line in open(dotconfig):
 422             m = self.re_arch.match(line)
 423             if m:
 424                 arch = m.group(1)
 425                 continue
 426             m = self.re_cpu.match(line)
 427             if m:
 428                 cpu = m.group(1)
 429
 430         if not arch:
 431             return None
 432
 433         # fix-up for aarch64
 434         if arch == 'arm' and cpu == 'armv8':
 435             arch = 'aarch64'
 436
 437         return CROSS_COMPILE.get(arch, None)
 438
 439     def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
 440         """Parse .config, defconfig, include/autoconf.mk for one config.
 441
 442         This function looks for the config options in the lines from
 443         defconfig, .config, and include/autoconf.mk in order to decide
 444         which action should be taken for this defconfig.
 445
 446         Arguments:
 447           config: CONFIG name to parse.
 448           dotconfig_lines: lines from the .config file.
 449           autoconf_lines: lines from the include/autoconf.mk file.
 450
 451         Returns:
 452           A tupple of the action for this defconfig and the line
 453           matched for the config.
 454         """
 455         not_set = '# %s is not set' % config
 456
 457         for line in dotconfig_lines:
 458             line = line.rstrip()
 459             if line.startswith(config + '=') or line == not_set:
 460                 old_val = line
 461                 break
 462         else:
 463             return (ACTION_NO_ENTRY, config)
 464
 465         for line in autoconf_lines:
 466             line = line.rstrip()
 467             if line.startswith(config + '='):
 468                 new_val = line
 469                 break
 470         else:
 471             new_val = not_set
 472
 473         if old_val == new_val:
 474             return (ACTION_NO_CHANGE, new_val)
 475
 476         # If this CONFIG is neither bool nor trisate
 477         if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
 478             # tools/scripts/define2mk.sed changes '1' to 'y'.
 479             # This is a problem if the CONFIG is int type.
 480             # Check the type in Kconfig and handle it correctly.
 481             if new_val[-2:] == '=y':
 482                 new_val = new_val[:-1] + '1'
 483
 484         return (ACTION_MOVE, new_val)
 485
 486     def update_dotconfig(self, defconfig):
 487         """Parse files for the config options and update the .config.
 488
 489         This function parses the generated .config and include/autoconf.mk
 490         searching the target options.
 491         Move the config option(s) to the .config as needed.
 492         Also, display the log to show what happened to the .config.
 493
 494         Arguments:
 495           defconfig: defconfig name.
 496         """
 497
 498         dotconfig_path = os.path.join(self.build_dir, '.config')
 499         autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk')
 500         results = []
 501
 502         with open(dotconfig_path) as f:
 503             dotconfig_lines = f.readlines()
 504
 505         with open(autoconf_path) as f:
 506             autoconf_lines = f.readlines()
 507
 508         for config in self.configs:
 509             result = self.parse_one_config(config, dotconfig_lines,
 510                                            autoconf_lines)
 511             results.append(result)
 512
 513         log = ''
 514
 515         for (action, value) in results:
 516             if action == ACTION_MOVE:
 517                 actlog = "Move '%s'" % value
 518                 log_color = COLOR_LIGHT_GREEN
 519             elif action == ACTION_NO_ENTRY:
 520                 actlog = "%s is not defined in Kconfig.  Do nothing." % value
 521                 log_color = COLOR_LIGHT_BLUE
 522             elif action == ACTION_NO_CHANGE:
 523                 actlog = "'%s' is the same as the define in Kconfig.  Do nothing." \
 524                          % value
 525                 log_color = COLOR_LIGHT_PURPLE
 526             else:
 527                 sys.exit("Internal Error. This should not happen.")
 528
 529             log += log_msg(self.options.color, log_color, defconfig, actlog)
 530
 531         # Some threads are running in parallel.
 532         # Print log in one shot to not mix up logs from different threads.
 533         print log,
 534         self.progress.show()
 535
 536         with open(dotconfig_path, 'a') as f:
 537             for (action, value) in results:
 538                 if action == ACTION_MOVE:
 539                     f.write(value + '\n')
 540
 541         os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf'))
 542         os.remove(autoconf_path)
 543
 544 class Slot:
 545
 546     """A slot to store a subprocess.
 547
 548     Each instance of this class handles one subprocess.
 549     This class is useful to control multiple threads
 550     for faster processing.
 551     """
 552
 553     def __init__(self, configs, options, progress, devnull, make_cmd):
 554         """Create a new process slot.
 555
 556         Arguments:
 557           configs: A list of CONFIGs to move.
 558           options: option flags.
 559           progress: A progress indicator.
 560           devnull: A file object of '/dev/null'.
 561           make_cmd: command name of GNU Make.
 562         """
 563         self.options = options
 564         self.progress = progress
 565         self.build_dir = tempfile.mkdtemp()
 566         self.devnull = devnull
 567         self.make_cmd = (make_cmd, 'O=' + self.build_dir)
 568         self.parser = KconfigParser(configs, options, progress, self.build_dir)
 569         self.state = STATE_IDLE
 570         self.failed_boards = []
 571
 572     def __del__(self):
 573         """Delete the working directory
 574
 575         This function makes sure the temporary directory is cleaned away
 576         even if Python suddenly dies due to error.  It should be done in here
 577         because it is guranteed the destructor is always invoked when the
 578         instance of the class gets unreferenced.
 579
 580         If the subprocess is still running, wait until it finishes.
 581         """
 582         if self.state != STATE_IDLE:
 583             while self.ps.poll() == None:
 584                 pass
 585         shutil.rmtree(self.build_dir)
 586
 587     def add(self, defconfig):
 588         """Assign a new subprocess for defconfig and add it to the slot.
 589
 590         If the slot is vacant, create a new subprocess for processing the
 591         given defconfig and add it to the slot.  Just returns False if
 592         the slot is occupied (i.e. the current subprocess is still running).
 593
 594         Arguments:
 595           defconfig: defconfig name.
 596
 597         Returns:
 598           Return True on success or False on failure
 599         """
 600         if self.state != STATE_IDLE:
 601             return False
 602         cmd = list(self.make_cmd)
 603         cmd.append(defconfig)
 604         self.ps = subprocess.Popen(cmd, stdout=self.devnull,
 605                                    stderr=subprocess.PIPE)
 606         self.defconfig = defconfig
 607         self.state = STATE_DEFCONFIG
 608         return True
 609
 610     def poll(self):
 611         """Check the status of the subprocess and handle it as needed.
 612
 613         Returns True if the slot is vacant (i.e. in idle state).
 614         If the configuration is successfully finished, assign a new
 615         subprocess to build include/autoconf.mk.
 616         If include/autoconf.mk is generated, invoke the parser to
 617         parse the .config and the include/autoconf.mk, and then set the
 618         slot back to the idle state.
 619
 620         Returns:
 621           Return True if the subprocess is terminated, False otherwise
 622         """
 623         if self.state == STATE_IDLE:
 624             return True
 625
 626         if self.ps.poll() == None:
 627             return False
 628
 629         if self.ps.poll() != 0:
 630             print >> sys.stderr, log_msg(self.options.color, COLOR_LIGHT_RED,
 631                                          self.defconfig, "Failed to process."),
 632             if self.options.verbose:
 633                 print >> sys.stderr, color_text(self.options.color,
 634                                                 COLOR_LIGHT_CYAN,
 635                                                 self.ps.stderr.read())
 636             self.progress.inc()
 637             self.progress.show()
 638             if self.options.exit_on_error:
 639                 sys.exit("Exit on error.")
 640             # If --exit-on-error flag is not set, skip this board and continue.
 641             # Record the failed board.
 642             self.failed_boards.append(self.defconfig)
 643             self.state = STATE_IDLE
 644             return True
 645
 646         if self.state == STATE_AUTOCONF:
 647             self.parser.update_dotconfig(self.defconfig)
 648
 649             """Save off the defconfig in a consistent way"""
 650             cmd = list(self.make_cmd)
 651             cmd.append('savedefconfig')
 652             self.ps = subprocess.Popen(cmd, stdout=self.devnull,
 653                                        stderr=subprocess.PIPE)
 654             self.state = STATE_SAVEDEFCONFIG
 655             return False
 656
 657         if self.state == STATE_SAVEDEFCONFIG:
 658             if not self.options.dry_run:
 659                 shutil.move(os.path.join(self.build_dir, 'defconfig'),
 660                             os.path.join('configs', self.defconfig))
 661             self.progress.inc()
 662             self.state = STATE_IDLE
 663             return True
 664
 665         self.cross_compile = self.parser.get_cross_compile()
 666         if self.cross_compile is None:
 667             print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW,
 668                                          self.defconfig,
 669                                          "Compiler is missing.  Do nothing."),
 670             self.progress.inc()
 671             self.progress.show()
 672             if self.options.exit_on_error:
 673                 sys.exit("Exit on error.")
 674             # If --exit-on-error flag is not set, skip this board and continue.
 675             # Record the failed board.
 676             self.failed_boards.append(self.defconfig)
 677             self.state = STATE_IDLE
 678             return True
 679
 680         cmd = list(self.make_cmd)
 681         if self.cross_compile:
 682             cmd.append('CROSS_COMPILE=%s' % self.cross_compile)
 683         cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
 684         cmd.append('include/config/auto.conf')
 685         self.ps = subprocess.Popen(cmd, stdout=self.devnull,
 686                                    stderr=subprocess.PIPE)
 687         self.state = STATE_AUTOCONF
 688         return False
 689
 690     def get_failed_boards(self):
 691         """Returns a list of failed boards (defconfigs) in this slot.
 692         """
 693         return self.failed_boards
 694
 695 class Slots:
 696
 697     """Controller of the array of subprocess slots."""
 698
 699     def __init__(self, configs, options, progress):
 700         """Create a new slots controller.
 701
 702         Arguments:
 703           configs: A list of CONFIGs to move.
 704           options: option flags.
 705           progress: A progress indicator.
 706         """
 707         self.options = options
 708         self.slots = []
 709         devnull = get_devnull()
 710         make_cmd = get_make_cmd()
 711         for i in range(options.jobs):
 712             self.slots.append(Slot(configs, options, progress, devnull,
 713                                    make_cmd))
 714
 715     def add(self, defconfig):
 716         """Add a new subprocess if a vacant slot is found.
 717
 718         Arguments:
 719           defconfig: defconfig name to be put into.
 720
 721         Returns:
 722           Return True on success or False on failure
 723         """
 724         for slot in self.slots:
 725             if slot.add(defconfig):
 726                 return True
 727         return False
 728
 729     def available(self):
 730         """Check if there is a vacant slot.
 731
 732         Returns:
 733           Return True if at lease one vacant slot is found, False otherwise.
 734         """
 735         for slot in self.slots:
 736             if slot.poll():
 737                 return True
 738         return False
 739
 740     def empty(self):
 741         """Check if all slots are vacant.
 742
 743         Returns:
 744           Return True if all the slots are vacant, False otherwise.
 745         """
 746         ret = True
 747         for slot in self.slots:
 748             if not slot.poll():
 749                 ret = False
 750         return ret
 751
 752     def show_failed_boards(self):
 753         """Display all of the failed boards (defconfigs)."""
 754         failed_boards = []
 755
 756         for slot in self.slots:
 757             failed_boards += slot.get_failed_boards()
 758
 759         if len(failed_boards) > 0:
 760             msg = [ "The following boards were not processed due to error:" ]
 761             msg += failed_boards
 762             for line in msg:
 763                 print >> sys.stderr, color_text(self.options.color,
 764                                                 COLOR_LIGHT_RED, line)
 765
 766             with open('moveconfig.failed', 'w') as f:
 767                 for board in failed_boards:
 768                     f.write(board + '\n')
 769
 770 def move_config(configs, options):
 771     """Move config options to defconfig files.
 772
 773     Arguments:
 774       configs: A list of CONFIGs to move.
 775       options: option flags
 776     """
 777     if len(configs) == 0:
 778         print 'Nothing to do. exit.'
 779         sys.exit(0)
 780
 781     print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs)
 782
 783     if options.defconfigs:
 784         defconfigs = [line.strip() for line in open(options.defconfigs)]
 785         for i, defconfig in enumerate(defconfigs):
 786             if not defconfig.endswith('_defconfig'):
 787                 defconfigs[i] = defconfig + '_defconfig'
 788             if not os.path.exists(os.path.join('configs', defconfigs[i])):
 789                 sys.exit('%s - defconfig does not exist. Stopping.' %
 790                          defconfigs[i])
 791     else:
 792         # All the defconfig files to be processed
 793         defconfigs = []
 794         for (dirpath, dirnames, filenames) in os.walk('configs'):
 795             dirpath = dirpath[len('configs') + 1:]
 796             for filename in fnmatch.filter(filenames, '*_defconfig'):
 797                 defconfigs.append(os.path.join(dirpath, filename))
 798
 799     progress = Progress(len(defconfigs))
 800     slots = Slots(configs, options, progress)
 801
 802     # Main loop to process defconfig files:
 803     #  Add a new subprocess into a vacant slot.
 804     #  Sleep if there is no available slot.
 805     for defconfig in defconfigs:
 806         while not slots.add(defconfig):
 807             while not slots.available():
 808                 # No available slot: sleep for a while
 809                 time.sleep(SLEEP_TIME)
 810
 811     # wait until all the subprocesses finish
 812     while not slots.empty():
 813         time.sleep(SLEEP_TIME)
 814
 815     progress.show()
 816     print ''
 817     slots.show_failed_boards()
 818
 819 def main():
 820     try:
 821         cpu_count = multiprocessing.cpu_count()
 822     except NotImplementedError:
 823         cpu_count = 1
 824
 825     parser = optparse.OptionParser()
 826     # Add options here
 827     parser.add_option('-c', '--color', action='store_true', default=False,
 828                       help='display the log in color')
 829     parser.add_option('-d', '--defconfigs', type='string',
 830                       help='a file containing a list of defconfigs to move')
 831     parser.add_option('-n', '--dry-run', action='store_true', default=False,
 832                       help='perform a trial run (show log with no changes)')
 833     parser.add_option('-e', '--exit-on-error', action='store_true',
 834                       default=False,
 835                       help='exit immediately on any error')
 836     parser.add_option('-H', '--headers-only', dest='cleanup_headers_only',
 837                       action='store_true', default=False,
 838                       help='only cleanup the headers')
 839     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 840                       help='the number of jobs to run simultaneously')
 841     parser.add_option('-v', '--verbose', action='store_true', default=False,
 842                       help='show any build errors as boards are built')
 843     parser.usage += ' CONFIG ...'
 844
 845     (options, configs) = parser.parse_args()
 846
 847     if len(configs) == 0:
 848         parser.print_usage()
 849         sys.exit(1)
 850
 851     # prefix the option name with CONFIG_ if missing
 852     configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config
 853                 for config in configs ]
 854
 855     check_top_directory()
 856
 857     check_clean_directory()
 858
 859     update_cross_compile(options.color)
 860
 861     if not options.cleanup_headers_only:
 862         move_config(configs, options)
 863
 864     cleanup_headers(configs, options.dry_run)
 865
 866 if __name__ == '__main__':
 867     main()