tools/moveconfig.py

   1 #!/usr/bin/env python2
   2 #
   3 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
   4 #
   5 # SPDX-License-Identifier:      GPL-2.0+
   6 #
   7
   8 """
   9 Move config options from headers to defconfig files.
  10
  11 Since Kconfig was introduced to U-Boot, we have worked on moving
  12 config options from headers to Kconfig (defconfig).
  13
  14 This tool intends to help this tremendous work.
  15
  16
  17 Usage
  18 -----
  19
  20 First, you must edit the Kconfig to add the menu entries for the configs
  21 you are moving.
  22
  23 And then run this tool giving CONFIG names you want to move.
  24 For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE,
  25 simply type as follows:
  26
  27   $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE
  28
  29 The tool walks through all the defconfig files and move the given CONFIGs.
  30
  31 The log is also displayed on the terminal.
  32
  33 The log is printed for each defconfig as follows:
  34
  35 <defconfig_name>
  36     <action1>
  37     <action2>
  38     <action3>
  39     ...
  40
  41 <defconfig_name> is the name of the defconfig.
  42
  43 <action*> shows what the tool did for that defconfig.
  44 It looks like one of the followings:
  45
  46  - Move 'CONFIG_... '
  47    This config option was moved to the defconfig
  48
  49  - CONFIG_... is not defined in Kconfig.  Do nothing.
  50    The entry for this CONFIG was not found in Kconfig.
  51    There are two common cases:
  52      - You forgot to create an entry for the CONFIG before running
  53        this tool, or made a typo in a CONFIG passed to this tool.
  54      - The entry was hidden due to unmet 'depends on'.
  55        This is correct behavior.
  56
  57  - 'CONFIG_...' is the same as the define in Kconfig.  Do nothing.
  58    The define in the config header matched the one in Kconfig.
  59    We do not need to touch it.
  60
  61  - Undefined.  Do nothing.
  62    This config option was not found in the config header.
  63    Nothing to do.
  64
  65  - Compiler is missing.  Do nothing.
  66    The compiler specified for this architecture was not found
  67    in your PATH environment.
  68    (If -e option is passed, the tool exits immediately.)
  69
  70  - Failed to process.
  71    An error occurred during processing this defconfig.  Skipped.
  72    (If -e option is passed, the tool exits immediately on error.)
  73
  74 Finally, you will be asked, Clean up headers? [y/n]:
  75
  76 If you say 'y' here, the unnecessary config defines are removed
  77 from the config headers (include/configs/*.h).
  78 It just uses the regex method, so you should not rely on it.
  79 Just in case, please do 'git diff' to see what happened.
  80
  81
  82 How does it work?
  83 -----------------
  84
  85 This tool runs configuration and builds include/autoconf.mk for every
  86 defconfig.  The config options defined in Kconfig appear in the .config
  87 file (unless they are hidden because of unmet dependency.)
  88 On the other hand, the config options defined by board headers are seen
  89 in include/autoconf.mk.  The tool looks for the specified options in both
  90 of them to decide the appropriate action for the options.  If the given
  91 config option is found in the .config, but its value does not match the
  92 one from the board header, the config option in the .config is replaced
  93 with the define in the board header.  Then, the .config is synced by
  94 "make savedefconfig" and the defconfig is updated with it.
  95
  96 For faster processing, this tool handles multi-threading.  It creates
  97 separate build directories where the out-of-tree build is run.  The
  98 temporary build directories are automatically created and deleted as
  99 needed.  The number of threads are chosen based on the number of the CPU
 100 cores of your system although you can change it via -j (--jobs) option.
 101
 102
 103 Toolchains
 104 ----------
 105
 106 Appropriate toolchain are necessary to generate include/autoconf.mk
 107 for all the architectures supported by U-Boot.  Most of them are available
 108 at the kernel.org site, some are not provided by kernel.org.
 109
 110 The default per-arch CROSS_COMPILE used by this tool is specified by
 111 the list below, CROSS_COMPILE.  You may wish to update the list to
 112 use your own.  Instead of modifying the list directly, you can give
 113 them via environments.
 114
 115
 116 Available options
 117 -----------------
 118
 119  -c, --color
 120    Surround each portion of the log with escape sequences to display it
 121    in color on the terminal.
 122
 123  -d, --defconfigs
 124   Specify a file containing a list of defconfigs to move
 125
 126  -n, --dry-run
 127    Perform a trial run that does not make any changes.  It is useful to
 128    see what is going to happen before one actually runs it.
 129
 130  -e, --exit-on-error
 131    Exit immediately if Make exits with a non-zero status while processing
 132    a defconfig file.
 133
 134  -s, --force-sync
 135    Do "make savedefconfig" forcibly for all the defconfig files.
 136    If not specified, "make savedefconfig" only occurs for cases
 137    where at least one CONFIG was moved.
 138
 139  -H, --headers-only
 140    Only cleanup the headers; skip the defconfig processing
 141
 142  -j, --jobs
 143    Specify the number of threads to run simultaneously.  If not specified,
 144    the number of threads is the same as the number of CPU cores.
 145
 146  -v, --verbose
 147    Show any build errors as boards are built
 148
 149 To see the complete list of supported options, run
 150
 151   $ tools/moveconfig.py -h
 152
 153 """
 154
 155 import filecmp
 156 import fnmatch
 157 import multiprocessing
 158 import optparse
 159 import os
 160 import re
 161 import shutil
 162 import subprocess
 163 import sys
 164 import tempfile
 165 import time
 166
 167 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
 168 SLEEP_TIME=0.03
 169
 170 # Here is the list of cross-tools I use.
 171 # Most of them are available at kernel.org
 172 # (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
 173 # arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
 174 # blackfin: http://sourceforge.net/projects/adi-toolchain/files/
 175 # nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz
 176 # nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545
 177 # sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu
 178 #
 179 # openrisc kernel.org toolchain is out of date, download latest one from
 180 # http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions
 181 CROSS_COMPILE = {
 182     'arc': 'arc-linux-',
 183     'aarch64': 'aarch64-linux-',
 184     'arm': 'arm-unknown-linux-gnueabi-',
 185     'avr32': 'avr32-linux-',
 186     'blackfin': 'bfin-elf-',
 187     'm68k': 'm68k-linux-',
 188     'microblaze': 'microblaze-linux-',
 189     'mips': 'mips-linux-',
 190     'nds32': 'nds32le-linux-',
 191     'nios2': 'nios2-linux-gnu-',
 192     'openrisc': 'or1k-elf-',
 193     'powerpc': 'powerpc-linux-',
 194     'sh': 'sh-linux-gnu-',
 195     'sparc': 'sparc-linux-',
 196     'x86': 'i386-linux-'
 197 }
 198
 199 STATE_IDLE = 0
 200 STATE_DEFCONFIG = 1
 201 STATE_AUTOCONF = 2
 202 STATE_SAVEDEFCONFIG = 3
 203
 204 ACTION_MOVE = 0
 205 ACTION_NO_ENTRY = 1
 206 ACTION_NO_CHANGE = 2
 207
 208 COLOR_BLACK        = '0;30'
 209 COLOR_RED          = '0;31'
 210 COLOR_GREEN        = '0;32'
 211 COLOR_BROWN        = '0;33'
 212 COLOR_BLUE         = '0;34'
 213 COLOR_PURPLE       = '0;35'
 214 COLOR_CYAN         = '0;36'
 215 COLOR_LIGHT_GRAY   = '0;37'
 216 COLOR_DARK_GRAY    = '1;30'
 217 COLOR_LIGHT_RED    = '1;31'
 218 COLOR_LIGHT_GREEN  = '1;32'
 219 COLOR_YELLOW       = '1;33'
 220 COLOR_LIGHT_BLUE   = '1;34'
 221 COLOR_LIGHT_PURPLE = '1;35'
 222 COLOR_LIGHT_CYAN   = '1;36'
 223 COLOR_WHITE        = '1;37'
 224
 225 ### helper functions ###
 226 def get_devnull():
 227     """Get the file object of '/dev/null' device."""
 228     try:
 229         devnull = subprocess.DEVNULL # py3k
 230     except AttributeError:
 231         devnull = open(os.devnull, 'wb')
 232     return devnull
 233
 234 def check_top_directory():
 235     """Exit if we are not at the top of source directory."""
 236     for f in ('README', 'Licenses'):
 237         if not os.path.exists(f):
 238             sys.exit('Please run at the top of source directory.')
 239
 240 def check_clean_directory():
 241     """Exit if the source tree is not clean."""
 242     for f in ('.config', 'include/config'):
 243         if os.path.exists(f):
 244             sys.exit("source tree is not clean, please run 'make mrproper'")
 245
 246 def get_make_cmd():
 247     """Get the command name of GNU Make.
 248
 249     U-Boot needs GNU Make for building, but the command name is not
 250     necessarily "make". (for example, "gmake" on FreeBSD).
 251     Returns the most appropriate command name on your system.
 252     """
 253     process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
 254     ret = process.communicate()
 255     if process.returncode:
 256         sys.exit('GNU Make not found')
 257     return ret[0].rstrip()
 258
 259 def color_text(color_enabled, color, string):
 260     """Return colored string."""
 261     if color_enabled:
 262         # LF should not be surrounded by the escape sequence.
 263         # Otherwise, additional whitespace or line-feed might be printed.
 264         return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
 265                            for s in string.split('\n') ])
 266     else:
 267         return string
 268
 269 def update_cross_compile(color_enabled):
 270     """Update per-arch CROSS_COMPILE via environment variables
 271
 272     The default CROSS_COMPILE values are available
 273     in the CROSS_COMPILE list above.
 274
 275     You can override them via environment variables
 276     CROSS_COMPILE_{ARCH}.
 277
 278     For example, if you want to override toolchain prefixes
 279     for ARM and PowerPC, you can do as follows in your shell:
 280
 281     export CROSS_COMPILE_ARM=...
 282     export CROSS_COMPILE_POWERPC=...
 283
 284     Then, this function checks if specified compilers really exist in your
 285     PATH environment.
 286     """
 287     archs = []
 288
 289     for arch in os.listdir('arch'):
 290         if os.path.exists(os.path.join('arch', arch, 'Makefile')):
 291             archs.append(arch)
 292
 293     # arm64 is a special case
 294     archs.append('aarch64')
 295
 296     for arch in archs:
 297         env = 'CROSS_COMPILE_' + arch.upper()
 298         cross_compile = os.environ.get(env)
 299         if not cross_compile:
 300             cross_compile = CROSS_COMPILE.get(arch, '')
 301
 302         for path in os.environ["PATH"].split(os.pathsep):
 303             gcc_path = os.path.join(path, cross_compile + 'gcc')
 304             if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK):
 305                 break
 306         else:
 307             print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW,
 308                  'warning: %sgcc: not found in PATH.  %s architecture boards will be skipped'
 309                                             % (cross_compile, arch))
 310             cross_compile = None
 311
 312         CROSS_COMPILE[arch] = cross_compile
 313
 314 def cleanup_one_header(header_path, patterns, dry_run):
 315     """Clean regex-matched lines away from a file.
 316
 317     Arguments:
 318       header_path: path to the cleaned file.
 319       patterns: list of regex patterns.  Any lines matching to these
 320                 patterns are deleted.
 321       dry_run: make no changes, but still display log.
 322     """
 323     with open(header_path) as f:
 324         lines = f.readlines()
 325
 326     matched = []
 327     for i, line in enumerate(lines):
 328         for pattern in patterns:
 329             m = pattern.search(line)
 330             if m:
 331                 print '%s: %s: %s' % (header_path, i + 1, line),
 332                 matched.append(i)
 333                 break
 334
 335     if dry_run or not matched:
 336         return
 337
 338     with open(header_path, 'w') as f:
 339         for i, line in enumerate(lines):
 340             if not i in matched:
 341                 f.write(line)
 342
 343 def cleanup_headers(configs, dry_run):
 344     """Delete config defines from board headers.
 345
 346     Arguments:
 347       configs: A list of CONFIGs to remove.
 348       dry_run: make no changes, but still display log.
 349     """
 350     while True:
 351         choice = raw_input('Clean up headers? [y/n]: ').lower()
 352         print choice
 353         if choice == 'y' or choice == 'n':
 354             break
 355
 356     if choice == 'n':
 357         return
 358
 359     patterns = []
 360     for config in configs:
 361         patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
 362         patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
 363
 364     for dir in 'include', 'arch', 'board':
 365         for (dirpath, dirnames, filenames) in os.walk(dir):
 366             for filename in filenames:
 367                 if not fnmatch.fnmatch(filename, '*~'):
 368                     cleanup_one_header(os.path.join(dirpath, filename),
 369                                        patterns, dry_run)
 370
 371 ### classes ###
 372 class Progress:
 373
 374     """Progress Indicator"""
 375
 376     def __init__(self, total):
 377         """Create a new progress indicator.
 378
 379         Arguments:
 380           total: A number of defconfig files to process.
 381         """
 382         self.current = 0
 383         self.total = total
 384
 385     def inc(self):
 386         """Increment the number of processed defconfig files."""
 387
 388         self.current += 1
 389
 390     def show(self):
 391         """Display the progress."""
 392         print ' %d defconfigs out of %d\r' % (self.current, self.total),
 393         sys.stdout.flush()
 394
 395 class KconfigParser:
 396
 397     """A parser of .config and include/autoconf.mk."""
 398
 399     re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
 400     re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
 401
 402     def __init__(self, configs, options, build_dir):
 403         """Create a new parser.
 404
 405         Arguments:
 406           configs: A list of CONFIGs to move.
 407           options: option flags.
 408           build_dir: Build directory.
 409         """
 410         self.configs = configs
 411         self.options = options
 412         self.dotconfig = os.path.join(build_dir, '.config')
 413         self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
 414         self.config_autoconf = os.path.join(build_dir, 'include', 'config',
 415                                             'auto.conf')
 416         self.defconfig = os.path.join(build_dir, 'defconfig')
 417
 418     def get_cross_compile(self):
 419         """Parse .config file and return CROSS_COMPILE.
 420
 421         Returns:
 422           A string storing the compiler prefix for the architecture.
 423           Return a NULL string for architectures that do not require
 424           compiler prefix (Sandbox and native build is the case).
 425           Return None if the specified compiler is missing in your PATH.
 426           Caller should distinguish '' and None.
 427         """
 428         arch = ''
 429         cpu = ''
 430         for line in open(self.dotconfig):
 431             m = self.re_arch.match(line)
 432             if m:
 433                 arch = m.group(1)
 434                 continue
 435             m = self.re_cpu.match(line)
 436             if m:
 437                 cpu = m.group(1)
 438
 439         if not arch:
 440             return None
 441
 442         # fix-up for aarch64
 443         if arch == 'arm' and cpu == 'armv8':
 444             arch = 'aarch64'
 445
 446         return CROSS_COMPILE.get(arch, None)
 447
 448     def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
 449         """Parse .config, defconfig, include/autoconf.mk for one config.
 450
 451         This function looks for the config options in the lines from
 452         defconfig, .config, and include/autoconf.mk in order to decide
 453         which action should be taken for this defconfig.
 454
 455         Arguments:
 456           config: CONFIG name to parse.
 457           dotconfig_lines: lines from the .config file.
 458           autoconf_lines: lines from the include/autoconf.mk file.
 459
 460         Returns:
 461           A tupple of the action for this defconfig and the line
 462           matched for the config.
 463         """
 464         not_set = '# %s is not set' % config
 465
 466         for line in dotconfig_lines:
 467             line = line.rstrip()
 468             if line.startswith(config + '=') or line == not_set:
 469                 old_val = line
 470                 break
 471         else:
 472             return (ACTION_NO_ENTRY, config)
 473
 474         for line in autoconf_lines:
 475             line = line.rstrip()
 476             if line.startswith(config + '='):
 477                 new_val = line
 478                 break
 479         else:
 480             new_val = not_set
 481
 482         if old_val == new_val:
 483             return (ACTION_NO_CHANGE, new_val)
 484
 485         # If this CONFIG is neither bool nor trisate
 486         if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
 487             # tools/scripts/define2mk.sed changes '1' to 'y'.
 488             # This is a problem if the CONFIG is int type.
 489             # Check the type in Kconfig and handle it correctly.
 490             if new_val[-2:] == '=y':
 491                 new_val = new_val[:-1] + '1'
 492
 493         return (ACTION_MOVE, new_val)
 494
 495     def update_dotconfig(self):
 496         """Parse files for the config options and update the .config.
 497
 498         This function parses the generated .config and include/autoconf.mk
 499         searching the target options.
 500         Move the config option(s) to the .config as needed.
 501
 502         Arguments:
 503           defconfig: defconfig name.
 504
 505         Returns:
 506           Return a tuple of (updated flag, log string).
 507           The "updated flag" is True if the .config was updated, False
 508           otherwise.  The "log string" shows what happend to the .config.
 509         """
 510
 511         results = []
 512         updated = False
 513
 514         with open(self.dotconfig) as f:
 515             dotconfig_lines = f.readlines()
 516
 517         with open(self.autoconf) as f:
 518             autoconf_lines = f.readlines()
 519
 520         for config in self.configs:
 521             result = self.parse_one_config(config, dotconfig_lines,
 522                                            autoconf_lines)
 523             results.append(result)
 524
 525         log = ''
 526
 527         for (action, value) in results:
 528             if action == ACTION_MOVE:
 529                 actlog = "Move '%s'" % value
 530                 log_color = COLOR_LIGHT_GREEN
 531             elif action == ACTION_NO_ENTRY:
 532                 actlog = "%s is not defined in Kconfig.  Do nothing." % value
 533                 log_color = COLOR_LIGHT_BLUE
 534             elif action == ACTION_NO_CHANGE:
 535                 actlog = "'%s' is the same as the define in Kconfig.  Do nothing." \
 536                          % value
 537                 log_color = COLOR_LIGHT_PURPLE
 538             else:
 539                 sys.exit("Internal Error. This should not happen.")
 540
 541             log += color_text(self.options.color, log_color, actlog) + '\n'
 542
 543         with open(self.dotconfig, 'a') as f:
 544             for (action, value) in results:
 545                 if action == ACTION_MOVE:
 546                     f.write(value + '\n')
 547                     updated = True
 548
 549         self.results = results
 550         os.remove(self.config_autoconf)
 551         os.remove(self.autoconf)
 552
 553         return (updated, log)
 554
 555     def check_defconfig(self):
 556         """Check the defconfig after savedefconfig
 557
 558         Returns:
 559           Return additional log if moved CONFIGs were removed again by
 560           'make savedefconfig'.
 561         """
 562
 563         log = ''
 564
 565         with open(self.defconfig) as f:
 566             defconfig_lines = f.readlines()
 567
 568         for (action, value) in self.results:
 569             if action != ACTION_MOVE:
 570                 continue
 571             if not value + '\n' in defconfig_lines:
 572                 log += color_text(self.options.color, COLOR_YELLOW,
 573                                   "'%s' was removed by savedefconfig.\n" %
 574                                   value)
 575
 576         return log
 577
 578 class Slot:
 579
 580     """A slot to store a subprocess.
 581
 582     Each instance of this class handles one subprocess.
 583     This class is useful to control multiple threads
 584     for faster processing.
 585     """
 586
 587     def __init__(self, configs, options, progress, devnull, make_cmd):
 588         """Create a new process slot.
 589
 590         Arguments:
 591           configs: A list of CONFIGs to move.
 592           options: option flags.
 593           progress: A progress indicator.
 594           devnull: A file object of '/dev/null'.
 595           make_cmd: command name of GNU Make.
 596         """
 597         self.options = options
 598         self.progress = progress
 599         self.build_dir = tempfile.mkdtemp()
 600         self.devnull = devnull
 601         self.make_cmd = (make_cmd, 'O=' + self.build_dir)
 602         self.parser = KconfigParser(configs, options, self.build_dir)
 603         self.state = STATE_IDLE
 604         self.failed_boards = []
 605
 606     def __del__(self):
 607         """Delete the working directory
 608
 609         This function makes sure the temporary directory is cleaned away
 610         even if Python suddenly dies due to error.  It should be done in here
 611         because it is guaranteed the destructor is always invoked when the
 612         instance of the class gets unreferenced.
 613
 614         If the subprocess is still running, wait until it finishes.
 615         """
 616         if self.state != STATE_IDLE:
 617             while self.ps.poll() == None:
 618                 pass
 619         shutil.rmtree(self.build_dir)
 620
 621     def add(self, defconfig):
 622         """Assign a new subprocess for defconfig and add it to the slot.
 623
 624         If the slot is vacant, create a new subprocess for processing the
 625         given defconfig and add it to the slot.  Just returns False if
 626         the slot is occupied (i.e. the current subprocess is still running).
 627
 628         Arguments:
 629           defconfig: defconfig name.
 630
 631         Returns:
 632           Return True on success or False on failure
 633         """
 634         if self.state != STATE_IDLE:
 635             return False
 636
 637         self.defconfig = defconfig
 638         self.log = ''
 639         self.do_defconfig()
 640         return True
 641
 642     def poll(self):
 643         """Check the status of the subprocess and handle it as needed.
 644
 645         Returns True if the slot is vacant (i.e. in idle state).
 646         If the configuration is successfully finished, assign a new
 647         subprocess to build include/autoconf.mk.
 648         If include/autoconf.mk is generated, invoke the parser to
 649         parse the .config and the include/autoconf.mk, moving
 650         config options to the .config as needed.
 651         If the .config was updated, run "make savedefconfig" to sync
 652         it, update the original defconfig, and then set the slot back
 653         to the idle state.
 654
 655         Returns:
 656           Return True if the subprocess is terminated, False otherwise
 657         """
 658         if self.state == STATE_IDLE:
 659             return True
 660
 661         if self.ps.poll() == None:
 662             return False
 663
 664         if self.ps.poll() != 0:
 665             self.handle_error()
 666         elif self.state == STATE_DEFCONFIG:
 667             self.do_autoconf()
 668         elif self.state == STATE_AUTOCONF:
 669             self.do_savedefconfig()
 670         elif self.state == STATE_SAVEDEFCONFIG:
 671             self.update_defconfig()
 672         else:
 673             sys.exit("Internal Error. This should not happen.")
 674
 675         return True if self.state == STATE_IDLE else False
 676
 677     def handle_error(self):
 678         """Handle error cases."""
 679
 680         self.log += color_text(self.options.color, COLOR_LIGHT_RED,
 681                                "Failed to process.\n")
 682         if self.options.verbose:
 683             self.log += color_text(self.options.color, COLOR_LIGHT_CYAN,
 684                                    self.ps.stderr.read())
 685         self.finish(False)
 686
 687     def do_defconfig(self):
 688         """Run 'make <board>_defconfig' to create the .config file."""
 689
 690         cmd = list(self.make_cmd)
 691         cmd.append(self.defconfig)
 692         self.ps = subprocess.Popen(cmd, stdout=self.devnull,
 693                                    stderr=subprocess.PIPE)
 694         self.state = STATE_DEFCONFIG
 695
 696     def do_autoconf(self):
 697         """Run 'make include/config/auto.conf'."""
 698
 699         self.cross_compile = self.parser.get_cross_compile()
 700         if self.cross_compile is None:
 701             self.log += color_text(self.options.color, COLOR_YELLOW,
 702                                    "Compiler is missing.  Do nothing.\n")
 703             self.finish(False)
 704             return
 705
 706         cmd = list(self.make_cmd)
 707         if self.cross_compile:
 708             cmd.append('CROSS_COMPILE=%s' % self.cross_compile)
 709         cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
 710         cmd.append('include/config/auto.conf')
 711         self.ps = subprocess.Popen(cmd, stdout=self.devnull,
 712                                    stderr=subprocess.PIPE)
 713         self.state = STATE_AUTOCONF
 714
 715     def do_savedefconfig(self):
 716         """Update the .config and run 'make savedefconfig'."""
 717
 718         (updated, log) = self.parser.update_dotconfig()
 719         self.log += log
 720
 721         if not self.options.force_sync and not updated:
 722             self.finish(True)
 723             return
 724         if updated:
 725             self.log += color_text(self.options.color, COLOR_LIGHT_GREEN,
 726                                    "Syncing by savedefconfig...\n")
 727         else:
 728             self.log += "Syncing by savedefconfig (forced by option)...\n"
 729
 730         cmd = list(self.make_cmd)
 731         cmd.append('savedefconfig')
 732         self.ps = subprocess.Popen(cmd, stdout=self.devnull,
 733                                    stderr=subprocess.PIPE)
 734         self.state = STATE_SAVEDEFCONFIG
 735
 736     def update_defconfig(self):
 737         """Update the input defconfig and go back to the idle state."""
 738
 739         self.log += self.parser.check_defconfig()
 740         orig_defconfig = os.path.join('configs', self.defconfig)
 741         new_defconfig = os.path.join(self.build_dir, 'defconfig')
 742         updated = not filecmp.cmp(orig_defconfig, new_defconfig)
 743
 744         if updated:
 745             self.log += color_text(self.options.color, COLOR_LIGHT_BLUE,
 746                                    "defconfig was updated.\n")
 747
 748         if not self.options.dry_run and updated:
 749             shutil.move(new_defconfig, orig_defconfig)
 750         self.finish(True)
 751
 752     def finish(self, success):
 753         """Display log along with progress and go to the idle state.
 754
 755         Arguments:
 756           success: Should be True when the defconfig was processed
 757                    successfully, or False when it fails.
 758         """
 759         # output at least 30 characters to hide the "* defconfigs out of *".
 760         log = self.defconfig.ljust(30) + '\n'
 761
 762         log += '\n'.join([ '    ' + s for s in self.log.split('\n') ])
 763         # Some threads are running in parallel.
 764         # Print log atomically to not mix up logs from different threads.
 765         print >> (sys.stdout if success else sys.stderr), log
 766
 767         if not success:
 768             if self.options.exit_on_error:
 769                 sys.exit("Exit on error.")
 770             # If --exit-on-error flag is not set, skip this board and continue.
 771             # Record the failed board.
 772             self.failed_boards.append(self.defconfig)
 773
 774         self.progress.inc()
 775         self.progress.show()
 776         self.state = STATE_IDLE
 777
 778     def get_failed_boards(self):
 779         """Returns a list of failed boards (defconfigs) in this slot.
 780         """
 781         return self.failed_boards
 782
 783 class Slots:
 784
 785     """Controller of the array of subprocess slots."""
 786
 787     def __init__(self, configs, options, progress):
 788         """Create a new slots controller.
 789
 790         Arguments:
 791           configs: A list of CONFIGs to move.
 792           options: option flags.
 793           progress: A progress indicator.
 794         """
 795         self.options = options
 796         self.slots = []
 797         devnull = get_devnull()
 798         make_cmd = get_make_cmd()
 799         for i in range(options.jobs):
 800             self.slots.append(Slot(configs, options, progress, devnull,
 801                                    make_cmd))
 802
 803     def add(self, defconfig):
 804         """Add a new subprocess if a vacant slot is found.
 805
 806         Arguments:
 807           defconfig: defconfig name to be put into.
 808
 809         Returns:
 810           Return True on success or False on failure
 811         """
 812         for slot in self.slots:
 813             if slot.add(defconfig):
 814                 return True
 815         return False
 816
 817     def available(self):
 818         """Check if there is a vacant slot.
 819
 820         Returns:
 821           Return True if at lease one vacant slot is found, False otherwise.
 822         """
 823         for slot in self.slots:
 824             if slot.poll():
 825                 return True
 826         return False
 827
 828     def empty(self):
 829         """Check if all slots are vacant.
 830
 831         Returns:
 832           Return True if all the slots are vacant, False otherwise.
 833         """
 834         ret = True
 835         for slot in self.slots:
 836             if not slot.poll():
 837                 ret = False
 838         return ret
 839
 840     def show_failed_boards(self):
 841         """Display all of the failed boards (defconfigs)."""
 842         failed_boards = []
 843
 844         for slot in self.slots:
 845             failed_boards += slot.get_failed_boards()
 846
 847         if len(failed_boards) > 0:
 848             msg = [ "The following boards were not processed due to error:" ]
 849             msg += failed_boards
 850             for line in msg:
 851                 print >> sys.stderr, color_text(self.options.color,
 852                                                 COLOR_LIGHT_RED, line)
 853
 854             with open('moveconfig.failed', 'w') as f:
 855                 for board in failed_boards:
 856                     f.write(board + '\n')
 857
 858 def move_config(configs, options):
 859     """Move config options to defconfig files.
 860
 861     Arguments:
 862       configs: A list of CONFIGs to move.
 863       options: option flags
 864     """
 865     if len(configs) == 0:
 866         if options.force_sync:
 867             print 'No CONFIG is specified. You are probably syncing defconfigs.',
 868         else:
 869             print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.',
 870     else:
 871         print 'Move ' + ', '.join(configs),
 872     print '(jobs: %d)\n' % options.jobs
 873
 874     if options.defconfigs:
 875         defconfigs = [line.strip() for line in open(options.defconfigs)]
 876         for i, defconfig in enumerate(defconfigs):
 877             if not defconfig.endswith('_defconfig'):
 878                 defconfigs[i] = defconfig + '_defconfig'
 879             if not os.path.exists(os.path.join('configs', defconfigs[i])):
 880                 sys.exit('%s - defconfig does not exist. Stopping.' %
 881                          defconfigs[i])
 882     else:
 883         # All the defconfig files to be processed
 884         defconfigs = []
 885         for (dirpath, dirnames, filenames) in os.walk('configs'):
 886             dirpath = dirpath[len('configs') + 1:]
 887             for filename in fnmatch.filter(filenames, '*_defconfig'):
 888                 defconfigs.append(os.path.join(dirpath, filename))
 889
 890     progress = Progress(len(defconfigs))
 891     slots = Slots(configs, options, progress)
 892
 893     # Main loop to process defconfig files:
 894     #  Add a new subprocess into a vacant slot.
 895     #  Sleep if there is no available slot.
 896     for defconfig in defconfigs:
 897         while not slots.add(defconfig):
 898             while not slots.available():
 899                 # No available slot: sleep for a while
 900                 time.sleep(SLEEP_TIME)
 901
 902     # wait until all the subprocesses finish
 903     while not slots.empty():
 904         time.sleep(SLEEP_TIME)
 905
 906     print ''
 907     slots.show_failed_boards()
 908
 909 def main():
 910     try:
 911         cpu_count = multiprocessing.cpu_count()
 912     except NotImplementedError:
 913         cpu_count = 1
 914
 915     parser = optparse.OptionParser()
 916     # Add options here
 917     parser.add_option('-c', '--color', action='store_true', default=False,
 918                       help='display the log in color')
 919     parser.add_option('-d', '--defconfigs', type='string',
 920                       help='a file containing a list of defconfigs to move')
 921     parser.add_option('-n', '--dry-run', action='store_true', default=False,
 922                       help='perform a trial run (show log with no changes)')
 923     parser.add_option('-e', '--exit-on-error', action='store_true',
 924                       default=False,
 925                       help='exit immediately on any error')
 926     parser.add_option('-s', '--force-sync', action='store_true', default=False,
 927                       help='force sync by savedefconfig')
 928     parser.add_option('-H', '--headers-only', dest='cleanup_headers_only',
 929                       action='store_true', default=False,
 930                       help='only cleanup the headers')
 931     parser.add_option('-j', '--jobs', type='int', default=cpu_count,
 932                       help='the number of jobs to run simultaneously')
 933     parser.add_option('-v', '--verbose', action='store_true', default=False,
 934                       help='show any build errors as boards are built')
 935     parser.usage += ' CONFIG ...'
 936
 937     (options, configs) = parser.parse_args()
 938
 939     if len(configs) == 0 and not options.force_sync:
 940         parser.print_usage()
 941         sys.exit(1)
 942
 943     # prefix the option name with CONFIG_ if missing
 944     configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config
 945                 for config in configs ]
 946
 947     check_top_directory()
 948
 949     check_clean_directory()
 950
 951     update_cross_compile(options.color)
 952
 953     if not options.cleanup_headers_only:
 954         move_config(configs, options)
 955
 956     if configs:
 957         cleanup_headers(configs, options.dry_run)
 958
 959 if __name__ == '__main__':
 960     main()