tools/buildman/func_test.py

   1 #
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4 # SPDX-License-Identifier:      GPL-2.0+
   5 #
   6
   7 import os
   8 import shutil
   9 import sys
  10 import tempfile
  11 import unittest
  12
  13 import board
  14 import bsettings
  15 import cmdline
  16 import command
  17 import control
  18 import gitutil
  19 import terminal
  20 import toolchain
  21
  22 settings_data = '''
  23 # Buildman settings file
  24
  25 [toolchain]
  26
  27 [toolchain-alias]
  28
  29 [make-flags]
  30 src=/home/sjg/c/src
  31 chroot=/home/sjg/c/chroot
  32 vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  33 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  34 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  35 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  36 '''
  37
  38 boards = [
  39     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  40     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  41     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  42     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  43 ]
  44
  45 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  46 39403bb patman: Use --no-pager' to stop git from forking a pager
  47 db6e6f2 patman: Remove the -a option
  48 f2ccf03 patman: Correct unit tests to run correctly
  49 1d097f9 patman: Fix indentation in terminal.py
  50 d073747 patman: Support the 'reverse' option for 'git log
  51 """
  52
  53 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  54 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  55 Date:   Fri Aug 22 19:12:41 2014 +0900
  56
  57     buildman: refactor help message
  58
  59     "buildman [options]" is displayed by default.
  60
  61     Append the rest of help messages to parser.usage
  62     instead of replacing it.
  63
  64     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  65     Drop it from the usage.
  66
  67     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  68 """,
  69 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  70 Author: Simon Glass <sjg@chromium.org>
  71 Date:   Thu Aug 14 16:48:25 2014 -0600
  72
  73     patman: Support the 'reverse' option for 'git log'
  74
  75     This option is currently not supported, but needs to be, for buildman to
  76     operate as expected.
  77
  78     Series-changes: 7
  79     - Add new patch to fix the 'reverse' bug
  80
  81     Series-version: 8
  82
  83     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  84     Reported-by: York Sun <yorksun@freescale.com>
  85     Signed-off-by: Simon Glass <sjg@chromium.org>
  86
  87 """,
  88 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  89 Author: Simon Glass <sjg@chromium.org>
  90 Date:   Sat Aug 9 11:44:32 2014 -0600
  91
  92     patman: Fix indentation in terminal.py
  93
  94     This code came from a different project with 2-character indentation. Fix
  95     it for U-Boot.
  96
  97     Series-changes: 6
  98     - Add new patch to fix indentation in teminal.py
  99
 100     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
 101     Signed-off-by: Simon Glass <sjg@chromium.org>
 102
 103 """,
 104 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 105 Author: Simon Glass <sjg@chromium.org>
 106 Date:   Sat Aug 9 11:08:24 2014 -0600
 107
 108     patman: Correct unit tests to run correctly
 109
 110     It seems that doctest behaves differently now, and some of the unit tests
 111     do not run. Adjust the tests to work correctly.
 112
 113      ./tools/patman/patman --test
 114     <unittest.result.TestResult run=10 errors=0 failures=0>
 115
 116     Series-changes: 6
 117     - Add new patch to fix patman unit tests
 118
 119     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 120
 121 """,
 122 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 123 Author: Simon Glass <sjg@chromium.org>
 124 Date:   Sat Aug 9 12:06:02 2014 -0600
 125
 126     patman: Remove the -a option
 127
 128     It seems that this is no longer needed, since checkpatch.pl will catch
 129     whitespace problems in patches. Also the option is not widely used, so
 130     it seems safe to just remove it.
 131
 132     Series-changes: 6
 133     - Add new patch to remove patman's -a option
 134
 135     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 136     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 137
 138 """,
 139 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 140 Author: Simon Glass <sjg@chromium.org>
 141 Date:   Thu Aug 14 21:50:52 2014 -0600
 142
 143     patman: Use --no-pager' to stop git from forking a pager
 144
 145 """,
 146 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 147 Author: Simon Glass <sjg@chromium.org>
 148 Date:   Fri Aug 22 15:57:39 2014 -0600
 149
 150     patman: Avoid changing the order of tags
 151
 152     patman collects tags that it sees in the commit and places them nicely
 153     sorted at the end of the patch. However, this is not really necessary and
 154     in fact is apparently not desirable.
 155
 156     Series-changes: 9
 157     - Add new patch to avoid changing the order of tags
 158
 159     Series-version: 9
 160
 161     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 162     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 163 """]
 164
 165 TEST_BRANCH = '__testbranch'
 166
 167 class TestFunctional(unittest.TestCase):
 168     """Functional test for buildman.
 169
 170     This aims to test from just below the invocation of buildman (parsing
 171     of arguments) to 'make' and 'git' invocation. It is not a true
 172     emd-to-end test, as it mocks git, make and the tool chain. But this
 173     makes it easier to detect when the builder is doing the wrong thing,
 174     since in many cases this test code will fail. For example, only a
 175     very limited subset of 'git' arguments is supported - anything
 176     unexpected will fail.
 177     """
 178     def setUp(self):
 179         self._base_dir = tempfile.mkdtemp()
 180         self._git_dir = os.path.join(self._base_dir, 'src')
 181         self._buildman_pathname = sys.argv[0]
 182         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 183         command.test_result = self._HandleCommand
 184         self.setupToolchains()
 185         self._toolchains.Add('arm-gcc', test=False)
 186         self._toolchains.Add('powerpc-gcc', test=False)
 187         bsettings.Setup(None)
 188         bsettings.AddFile(settings_data)
 189         self._boards = board.Boards()
 190         for brd in boards:
 191             self._boards.AddBoard(board.Board(*brd))
 192
 193         # Directories where the source been cloned
 194         self._clone_dirs = []
 195         self._commits = len(commit_shortlog.splitlines()) + 1
 196         self._total_builds = self._commits * len(boards)
 197
 198         # Number of calls to make
 199         self._make_calls = 0
 200
 201         # Map of [board, commit] to error messages
 202         self._error = {}
 203
 204         self._test_branch = TEST_BRANCH
 205
 206         # Avoid sending any output and clear all terminal output
 207         terminal.SetPrintTestMode()
 208         terminal.GetPrintTestLines()
 209
 210     def tearDown(self):
 211         shutil.rmtree(self._base_dir)
 212
 213     def setupToolchains(self):
 214         self._toolchains = toolchain.Toolchains()
 215         self._toolchains.Add('gcc', test=False)
 216
 217     def _RunBuildman(self, *args):
 218         return command.RunPipe([[self._buildman_pathname] + list(args)],
 219                 capture=True, capture_stderr=True)
 220
 221     def _RunControl(self, *args, **kwargs):
 222         sys.argv = [sys.argv[0]] + list(args)
 223         options, args = cmdline.ParseArgs()
 224         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 225                 make_func=self._HandleMake, boards=self._boards,
 226                 clean_dir=kwargs.get('clean_dir', True))
 227         self._builder = control.builder
 228         return result
 229
 230     def testFullHelp(self):
 231         command.test_result = None
 232         result = self._RunBuildman('-H')
 233         help_file = os.path.join(self._buildman_dir, 'README')
 234         # Remove possible extraneous strings
 235         extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
 236         gothelp = result.stdout.replace(extra, '')
 237         self.assertEqual(len(gothelp), os.path.getsize(help_file))
 238         self.assertEqual(0, len(result.stderr))
 239         self.assertEqual(0, result.return_code)
 240
 241     def testHelp(self):
 242         command.test_result = None
 243         result = self._RunBuildman('-h')
 244         help_file = os.path.join(self._buildman_dir, 'README')
 245         self.assertTrue(len(result.stdout) > 1000)
 246         self.assertEqual(0, len(result.stderr))
 247         self.assertEqual(0, result.return_code)
 248
 249     def testGitSetup(self):
 250         """Test gitutils.Setup(), from outside the module itself"""
 251         command.test_result = command.CommandResult(return_code=1)
 252         gitutil.Setup()
 253         self.assertEqual(gitutil.use_no_decorate, False)
 254
 255         command.test_result = command.CommandResult(return_code=0)
 256         gitutil.Setup()
 257         self.assertEqual(gitutil.use_no_decorate, True)
 258
 259     def _HandleCommandGitLog(self, args):
 260         if args[-1] == '--':
 261             args = args[:-1]
 262         if '-n0' in args:
 263             return command.CommandResult(return_code=0)
 264         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 265             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 266         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 267             if args[-1] == self._test_branch:
 268                 count = int(args[3][2:])
 269                 return command.CommandResult(return_code=0,
 270                                             stdout=''.join(commit_log[:count]))
 271
 272         # Not handled, so abort
 273         print 'git log', args
 274         sys.exit(1)
 275
 276     def _HandleCommandGitConfig(self, args):
 277         config = args[0]
 278         if config == 'sendemail.aliasesfile':
 279             return command.CommandResult(return_code=0)
 280         elif config.startswith('branch.badbranch'):
 281             return command.CommandResult(return_code=1)
 282         elif config == 'branch.%s.remote' % self._test_branch:
 283             return command.CommandResult(return_code=0, stdout='upstream\n')
 284         elif config == 'branch.%s.merge' % self._test_branch:
 285             return command.CommandResult(return_code=0,
 286                                          stdout='refs/heads/master\n')
 287
 288         # Not handled, so abort
 289         print 'git config', args
 290         sys.exit(1)
 291
 292     def _HandleCommandGit(self, in_args):
 293         """Handle execution of a git command
 294
 295         This uses a hacked-up parser.
 296
 297         Args:
 298             in_args: Arguments after 'git' from the command line
 299         """
 300         git_args = []           # Top-level arguments to git itself
 301         sub_cmd = None          # Git sub-command selected
 302         args = []               # Arguments to the git sub-command
 303         for arg in in_args:
 304             if sub_cmd:
 305                 args.append(arg)
 306             elif arg[0] == '-':
 307                 git_args.append(arg)
 308             else:
 309                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 310                     git_args.append(arg)
 311                 else:
 312                     sub_cmd = arg
 313         if sub_cmd == 'config':
 314             return self._HandleCommandGitConfig(args)
 315         elif sub_cmd == 'log':
 316             return self._HandleCommandGitLog(args)
 317         elif sub_cmd == 'clone':
 318             return command.CommandResult(return_code=0)
 319         elif sub_cmd == 'checkout':
 320             return command.CommandResult(return_code=0)
 321
 322         # Not handled, so abort
 323         print 'git', git_args, sub_cmd, args
 324         sys.exit(1)
 325
 326     def _HandleCommandNm(self, args):
 327         return command.CommandResult(return_code=0)
 328
 329     def _HandleCommandObjdump(self, args):
 330         return command.CommandResult(return_code=0)
 331
 332     def _HandleCommandSize(self, args):
 333         return command.CommandResult(return_code=0)
 334
 335     def _HandleCommand(self, **kwargs):
 336         """Handle a command execution.
 337
 338         The command is in kwargs['pipe-list'], as a list of pipes, each a
 339         list of commands. The command should be emulated as required for
 340         testing purposes.
 341
 342         Returns:
 343             A CommandResult object
 344         """
 345         pipe_list = kwargs['pipe_list']
 346         wc = False
 347         if len(pipe_list) != 1:
 348             if pipe_list[1] == ['wc', '-l']:
 349                 wc = True
 350             else:
 351                 print 'invalid pipe', kwargs
 352                 sys.exit(1)
 353         cmd = pipe_list[0][0]
 354         args = pipe_list[0][1:]
 355         result = None
 356         if cmd == 'git':
 357             result = self._HandleCommandGit(args)
 358         elif cmd == './scripts/show-gnu-make':
 359             return command.CommandResult(return_code=0, stdout='make')
 360         elif cmd.endswith('nm'):
 361             return self._HandleCommandNm(args)
 362         elif cmd.endswith('objdump'):
 363             return self._HandleCommandObjdump(args)
 364         elif cmd.endswith( 'size'):
 365             return self._HandleCommandSize(args)
 366
 367         if not result:
 368             # Not handled, so abort
 369             print 'unknown command', kwargs
 370             sys.exit(1)
 371
 372         if wc:
 373             result.stdout = len(result.stdout.splitlines())
 374         return result
 375
 376     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 377         """Handle execution of 'make'
 378
 379         Args:
 380             commit: Commit object that is being built
 381             brd: Board object that is being built
 382             stage: Stage that we are at (mrproper, config, build)
 383             cwd: Directory where make should be run
 384             args: Arguments to pass to make
 385             kwargs: Arguments to pass to command.RunPipe()
 386         """
 387         self._make_calls += 1
 388         if stage == 'mrproper':
 389             return command.CommandResult(return_code=0)
 390         elif stage == 'config':
 391             return command.CommandResult(return_code=0,
 392                     combined='Test configuration complete')
 393         elif stage == 'build':
 394             stderr = ''
 395             if type(commit) is not str:
 396                 stderr = self._error.get((brd.target, commit.sequence))
 397             if stderr:
 398                 return command.CommandResult(return_code=1, stderr=stderr)
 399             return command.CommandResult(return_code=0)
 400
 401         # Not handled, so abort
 402         print 'make', stage
 403         sys.exit(1)
 404
 405     # Example function to print output lines
 406     def print_lines(self, lines):
 407         print len(lines)
 408         for line in lines:
 409             print line
 410         #self.print_lines(terminal.GetPrintTestLines())
 411
 412     def testNoBoards(self):
 413         """Test that buildman aborts when there are no boards"""
 414         self._boards = board.Boards()
 415         with self.assertRaises(SystemExit):
 416             self._RunControl()
 417
 418     def testCurrentSource(self):
 419         """Very simple test to invoke buildman on the current source"""
 420         self.setupToolchains();
 421         self._RunControl()
 422         lines = terminal.GetPrintTestLines()
 423         self.assertIn('Building current source for %d boards' % len(boards),
 424                       lines[0].text)
 425
 426     def testBadBranch(self):
 427         """Test that we can detect an invalid branch"""
 428         with self.assertRaises(ValueError):
 429             self._RunControl('-b', 'badbranch')
 430
 431     def testBadToolchain(self):
 432         """Test that missing toolchains are detected"""
 433         self.setupToolchains();
 434         ret_code = self._RunControl('-b', TEST_BRANCH)
 435         lines = terminal.GetPrintTestLines()
 436
 437         # Buildman always builds the upstream commit as well
 438         self.assertIn('Building %d commits for %d boards' %
 439                 (self._commits, len(boards)), lines[0].text)
 440         self.assertEqual(self._builder.count, self._total_builds)
 441
 442         # Only sandbox should succeed, the others don't have toolchains
 443         self.assertEqual(self._builder.fail,
 444                          self._total_builds - self._commits)
 445         self.assertEqual(ret_code, 128)
 446
 447         for commit in range(self._commits):
 448             for board in self._boards.GetList():
 449                 if board.arch != 'sandbox':
 450                   errfile = self._builder.GetErrFile(commit, board.target)
 451                   fd = open(errfile)
 452                   self.assertEqual(fd.readlines(),
 453                           ['No tool chain for %s\n' % board.arch])
 454                   fd.close()
 455
 456     def testBranch(self):
 457         """Test building a branch with all toolchains present"""
 458         self._RunControl('-b', TEST_BRANCH)
 459         self.assertEqual(self._builder.count, self._total_builds)
 460         self.assertEqual(self._builder.fail, 0)
 461
 462     def testCount(self):
 463         """Test building a specific number of commitst"""
 464         self._RunControl('-b', TEST_BRANCH, '-c2')
 465         self.assertEqual(self._builder.count, 2 * len(boards))
 466         self.assertEqual(self._builder.fail, 0)
 467         # Each board has a mrproper, config, and then one make per commit
 468         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
 469
 470     def testIncremental(self):
 471         """Test building a branch twice - the second time should do nothing"""
 472         self._RunControl('-b', TEST_BRANCH)
 473
 474         # Each board has a mrproper, config, and then one make per commit
 475         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 476         self._make_calls = 0
 477         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 478         self.assertEqual(self._make_calls, 0)
 479         self.assertEqual(self._builder.count, self._total_builds)
 480         self.assertEqual(self._builder.fail, 0)
 481
 482     def testForceBuild(self):
 483         """The -f flag should force a rebuild"""
 484         self._RunControl('-b', TEST_BRANCH)
 485         self._make_calls = 0
 486         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
 487         # Each board has a mrproper, config, and then one make per commit
 488         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 489
 490     def testForceReconfigure(self):
 491         """The -f flag should force a rebuild"""
 492         self._RunControl('-b', TEST_BRANCH, '-C')
 493         # Each commit has a mrproper, config and make
 494         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 495
 496     def testErrors(self):
 497         """Test handling of build errors"""
 498         self._error['board2', 1] = 'fred\n'
 499         self._RunControl('-b', TEST_BRANCH)
 500         self.assertEqual(self._builder.count, self._total_builds)
 501         self.assertEqual(self._builder.fail, 1)
 502
 503         # Remove the error. This should have no effect since the commit will
 504         # not be rebuilt
 505         del self._error['board2', 1]
 506         self._make_calls = 0
 507         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 508         self.assertEqual(self._builder.count, self._total_builds)
 509         self.assertEqual(self._make_calls, 0)
 510         self.assertEqual(self._builder.fail, 1)
 511
 512         # Now use the -F flag to force rebuild of the bad commit
 513         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
 514         self.assertEqual(self._builder.count, self._total_builds)
 515         self.assertEqual(self._builder.fail, 0)
 516         self.assertEqual(self._make_calls, 3)
 517
 518     def testBranchWithSlash(self):
 519         """Test building a branch with a '/' in the name"""
 520         self._test_branch = '/__dev/__testbranch'
 521         self._RunControl('-b', self._test_branch, clean_dir=False)
 522         self.assertEqual(self._builder.count, self._total_builds)
 523         self.assertEqual(self._builder.fail, 0)