tools/buildman/func_test.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4
   5 import os
   6 import shutil
   7 import sys
   8 import tempfile
   9 import unittest
  10
  11 import board
  12 import bsettings
  13 import cmdline
  14 import command
  15 import control
  16 import gitutil
  17 import terminal
  18 import toolchain
  19 import tools
  20
  21 settings_data = '''
  22 # Buildman settings file
  23
  24 [toolchain]
  25
  26 [toolchain-alias]
  27
  28 [make-flags]
  29 src=/home/sjg/c/src
  30 chroot=/home/sjg/c/chroot
  31 vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  32 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  33 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  34 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  35 '''
  36
  37 boards = [
  38     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  39     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  40     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  41     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  42 ]
  43
  44 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  45 39403bb patman: Use --no-pager' to stop git from forking a pager
  46 db6e6f2 patman: Remove the -a option
  47 f2ccf03 patman: Correct unit tests to run correctly
  48 1d097f9 patman: Fix indentation in terminal.py
  49 d073747 patman: Support the 'reverse' option for 'git log
  50 """
  51
  52 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  53 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  54 Date:   Fri Aug 22 19:12:41 2014 +0900
  55
  56     buildman: refactor help message
  57
  58     "buildman [options]" is displayed by default.
  59
  60     Append the rest of help messages to parser.usage
  61     instead of replacing it.
  62
  63     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  64     Drop it from the usage.
  65
  66     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  67 """,
  68 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  69 Author: Simon Glass <sjg@chromium.org>
  70 Date:   Thu Aug 14 16:48:25 2014 -0600
  71
  72     patman: Support the 'reverse' option for 'git log'
  73
  74     This option is currently not supported, but needs to be, for buildman to
  75     operate as expected.
  76
  77     Series-changes: 7
  78     - Add new patch to fix the 'reverse' bug
  79
  80     Series-version: 8
  81
  82     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  83     Reported-by: York Sun <yorksun@freescale.com>
  84     Signed-off-by: Simon Glass <sjg@chromium.org>
  85
  86 """,
  87 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  88 Author: Simon Glass <sjg@chromium.org>
  89 Date:   Sat Aug 9 11:44:32 2014 -0600
  90
  91     patman: Fix indentation in terminal.py
  92
  93     This code came from a different project with 2-character indentation. Fix
  94     it for U-Boot.
  95
  96     Series-changes: 6
  97     - Add new patch to fix indentation in teminal.py
  98
  99     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
 100     Signed-off-by: Simon Glass <sjg@chromium.org>
 101
 102 """,
 103 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 104 Author: Simon Glass <sjg@chromium.org>
 105 Date:   Sat Aug 9 11:08:24 2014 -0600
 106
 107     patman: Correct unit tests to run correctly
 108
 109     It seems that doctest behaves differently now, and some of the unit tests
 110     do not run. Adjust the tests to work correctly.
 111
 112      ./tools/patman/patman --test
 113     <unittest.result.TestResult run=10 errors=0 failures=0>
 114
 115     Series-changes: 6
 116     - Add new patch to fix patman unit tests
 117
 118     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 119
 120 """,
 121 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 122 Author: Simon Glass <sjg@chromium.org>
 123 Date:   Sat Aug 9 12:06:02 2014 -0600
 124
 125     patman: Remove the -a option
 126
 127     It seems that this is no longer needed, since checkpatch.pl will catch
 128     whitespace problems in patches. Also the option is not widely used, so
 129     it seems safe to just remove it.
 130
 131     Series-changes: 6
 132     - Add new patch to remove patman's -a option
 133
 134     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 135     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 136
 137 """,
 138 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 139 Author: Simon Glass <sjg@chromium.org>
 140 Date:   Thu Aug 14 21:50:52 2014 -0600
 141
 142     patman: Use --no-pager' to stop git from forking a pager
 143
 144 """,
 145 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 146 Author: Simon Glass <sjg@chromium.org>
 147 Date:   Fri Aug 22 15:57:39 2014 -0600
 148
 149     patman: Avoid changing the order of tags
 150
 151     patman collects tags that it sees in the commit and places them nicely
 152     sorted at the end of the patch. However, this is not really necessary and
 153     in fact is apparently not desirable.
 154
 155     Series-changes: 9
 156     - Add new patch to avoid changing the order of tags
 157
 158     Series-version: 9
 159
 160     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 161     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 162 """]
 163
 164 TEST_BRANCH = '__testbranch'
 165
 166 class TestFunctional(unittest.TestCase):
 167     """Functional test for buildman.
 168
 169     This aims to test from just below the invocation of buildman (parsing
 170     of arguments) to 'make' and 'git' invocation. It is not a true
 171     emd-to-end test, as it mocks git, make and the tool chain. But this
 172     makes it easier to detect when the builder is doing the wrong thing,
 173     since in many cases this test code will fail. For example, only a
 174     very limited subset of 'git' arguments is supported - anything
 175     unexpected will fail.
 176     """
 177     def setUp(self):
 178         self._base_dir = tempfile.mkdtemp()
 179         self._output_dir = tempfile.mkdtemp()
 180         self._git_dir = os.path.join(self._base_dir, 'src')
 181         self._buildman_pathname = sys.argv[0]
 182         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 183         command.test_result = self._HandleCommand
 184         self.setupToolchains()
 185         self._toolchains.Add('arm-gcc', test=False)
 186         self._toolchains.Add('powerpc-gcc', test=False)
 187         bsettings.Setup(None)
 188         bsettings.AddFile(settings_data)
 189         self._boards = board.Boards()
 190         for brd in boards:
 191             self._boards.AddBoard(board.Board(*brd))
 192
 193         # Directories where the source been cloned
 194         self._clone_dirs = []
 195         self._commits = len(commit_shortlog.splitlines()) + 1
 196         self._total_builds = self._commits * len(boards)
 197
 198         # Number of calls to make
 199         self._make_calls = 0
 200
 201         # Map of [board, commit] to error messages
 202         self._error = {}
 203
 204         self._test_branch = TEST_BRANCH
 205
 206         # Avoid sending any output and clear all terminal output
 207         terminal.SetPrintTestMode()
 208         terminal.GetPrintTestLines()
 209
 210     def tearDown(self):
 211         shutil.rmtree(self._base_dir)
 212         #shutil.rmtree(self._output_dir)
 213
 214     def setupToolchains(self):
 215         self._toolchains = toolchain.Toolchains()
 216         self._toolchains.Add('gcc', test=False)
 217
 218     def _RunBuildman(self, *args):
 219         return command.RunPipe([[self._buildman_pathname] + list(args)],
 220                 capture=True, capture_stderr=True)
 221
 222     def _RunControl(self, *args, clean_dir=False, boards=None):
 223         sys.argv = [sys.argv[0]] + list(args)
 224         options, args = cmdline.ParseArgs()
 225         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 226                 make_func=self._HandleMake, boards=boards or self._boards,
 227                 clean_dir=clean_dir)
 228         self._builder = control.builder
 229         return result
 230
 231     def testFullHelp(self):
 232         command.test_result = None
 233         result = self._RunBuildman('-H')
 234         help_file = os.path.join(self._buildman_dir, 'README')
 235         # Remove possible extraneous strings
 236         extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
 237         gothelp = result.stdout.replace(extra, '')
 238         self.assertEqual(len(gothelp), os.path.getsize(help_file))
 239         self.assertEqual(0, len(result.stderr))
 240         self.assertEqual(0, result.return_code)
 241
 242     def testHelp(self):
 243         command.test_result = None
 244         result = self._RunBuildman('-h')
 245         help_file = os.path.join(self._buildman_dir, 'README')
 246         self.assertTrue(len(result.stdout) > 1000)
 247         self.assertEqual(0, len(result.stderr))
 248         self.assertEqual(0, result.return_code)
 249
 250     def testGitSetup(self):
 251         """Test gitutils.Setup(), from outside the module itself"""
 252         command.test_result = command.CommandResult(return_code=1)
 253         gitutil.Setup()
 254         self.assertEqual(gitutil.use_no_decorate, False)
 255
 256         command.test_result = command.CommandResult(return_code=0)
 257         gitutil.Setup()
 258         self.assertEqual(gitutil.use_no_decorate, True)
 259
 260     def _HandleCommandGitLog(self, args):
 261         if args[-1] == '--':
 262             args = args[:-1]
 263         if '-n0' in args:
 264             return command.CommandResult(return_code=0)
 265         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 266             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 267         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 268             if args[-1] == self._test_branch:
 269                 count = int(args[3][2:])
 270                 return command.CommandResult(return_code=0,
 271                                             stdout=''.join(commit_log[:count]))
 272
 273         # Not handled, so abort
 274         print('git log', args)
 275         sys.exit(1)
 276
 277     def _HandleCommandGitConfig(self, args):
 278         config = args[0]
 279         if config == 'sendemail.aliasesfile':
 280             return command.CommandResult(return_code=0)
 281         elif config.startswith('branch.badbranch'):
 282             return command.CommandResult(return_code=1)
 283         elif config == 'branch.%s.remote' % self._test_branch:
 284             return command.CommandResult(return_code=0, stdout='upstream\n')
 285         elif config == 'branch.%s.merge' % self._test_branch:
 286             return command.CommandResult(return_code=0,
 287                                          stdout='refs/heads/master\n')
 288
 289         # Not handled, so abort
 290         print('git config', args)
 291         sys.exit(1)
 292
 293     def _HandleCommandGit(self, in_args):
 294         """Handle execution of a git command
 295
 296         This uses a hacked-up parser.
 297
 298         Args:
 299             in_args: Arguments after 'git' from the command line
 300         """
 301         git_args = []           # Top-level arguments to git itself
 302         sub_cmd = None          # Git sub-command selected
 303         args = []               # Arguments to the git sub-command
 304         for arg in in_args:
 305             if sub_cmd:
 306                 args.append(arg)
 307             elif arg[0] == '-':
 308                 git_args.append(arg)
 309             else:
 310                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 311                     git_args.append(arg)
 312                 else:
 313                     sub_cmd = arg
 314         if sub_cmd == 'config':
 315             return self._HandleCommandGitConfig(args)
 316         elif sub_cmd == 'log':
 317             return self._HandleCommandGitLog(args)
 318         elif sub_cmd == 'clone':
 319             return command.CommandResult(return_code=0)
 320         elif sub_cmd == 'checkout':
 321             return command.CommandResult(return_code=0)
 322
 323         # Not handled, so abort
 324         print('git', git_args, sub_cmd, args)
 325         sys.exit(1)
 326
 327     def _HandleCommandNm(self, args):
 328         return command.CommandResult(return_code=0)
 329
 330     def _HandleCommandObjdump(self, args):
 331         return command.CommandResult(return_code=0)
 332
 333     def _HandleCommandObjcopy(self, args):
 334         return command.CommandResult(return_code=0)
 335
 336     def _HandleCommandSize(self, args):
 337         return command.CommandResult(return_code=0)
 338
 339     def _HandleCommand(self, **kwargs):
 340         """Handle a command execution.
 341
 342         The command is in kwargs['pipe-list'], as a list of pipes, each a
 343         list of commands. The command should be emulated as required for
 344         testing purposes.
 345
 346         Returns:
 347             A CommandResult object
 348         """
 349         pipe_list = kwargs['pipe_list']
 350         wc = False
 351         if len(pipe_list) != 1:
 352             if pipe_list[1] == ['wc', '-l']:
 353                 wc = True
 354             else:
 355                 print('invalid pipe', kwargs)
 356                 sys.exit(1)
 357         cmd = pipe_list[0][0]
 358         args = pipe_list[0][1:]
 359         result = None
 360         if cmd == 'git':
 361             result = self._HandleCommandGit(args)
 362         elif cmd == './scripts/show-gnu-make':
 363             return command.CommandResult(return_code=0, stdout='make')
 364         elif cmd.endswith('nm'):
 365             return self._HandleCommandNm(args)
 366         elif cmd.endswith('objdump'):
 367             return self._HandleCommandObjdump(args)
 368         elif cmd.endswith('objcopy'):
 369             return self._HandleCommandObjcopy(args)
 370         elif cmd.endswith( 'size'):
 371             return self._HandleCommandSize(args)
 372
 373         if not result:
 374             # Not handled, so abort
 375             print('unknown command', kwargs)
 376             sys.exit(1)
 377
 378         if wc:
 379             result.stdout = len(result.stdout.splitlines())
 380         return result
 381
 382     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 383         """Handle execution of 'make'
 384
 385         Args:
 386             commit: Commit object that is being built
 387             brd: Board object that is being built
 388             stage: Stage that we are at (mrproper, config, build)
 389             cwd: Directory where make should be run
 390             args: Arguments to pass to make
 391             kwargs: Arguments to pass to command.RunPipe()
 392         """
 393         self._make_calls += 1
 394         if stage == 'mrproper':
 395             return command.CommandResult(return_code=0)
 396         elif stage == 'config':
 397             return command.CommandResult(return_code=0,
 398                     combined='Test configuration complete')
 399         elif stage == 'build':
 400             stderr = ''
 401             out_dir = ''
 402             for arg in args:
 403                 if arg.startswith('O='):
 404                     out_dir = arg[2:]
 405             fname = os.path.join(cwd or '', out_dir, 'u-boot')
 406             tools.WriteFile(fname, b'U-Boot')
 407             if type(commit) is not str:
 408                 stderr = self._error.get((brd.target, commit.sequence))
 409             if stderr:
 410                 return command.CommandResult(return_code=1, stderr=stderr)
 411             return command.CommandResult(return_code=0)
 412
 413         # Not handled, so abort
 414         print('make', stage)
 415         sys.exit(1)
 416
 417     # Example function to print output lines
 418     def print_lines(self, lines):
 419         print(len(lines))
 420         for line in lines:
 421             print(line)
 422         #self.print_lines(terminal.GetPrintTestLines())
 423
 424     def testNoBoards(self):
 425         """Test that buildman aborts when there are no boards"""
 426         self._boards = board.Boards()
 427         with self.assertRaises(SystemExit):
 428             self._RunControl()
 429
 430     def testCurrentSource(self):
 431         """Very simple test to invoke buildman on the current source"""
 432         self.setupToolchains();
 433         self._RunControl('-o', self._output_dir)
 434         lines = terminal.GetPrintTestLines()
 435         self.assertIn('Building current source for %d boards' % len(boards),
 436                       lines[0].text)
 437
 438     def testBadBranch(self):
 439         """Test that we can detect an invalid branch"""
 440         with self.assertRaises(ValueError):
 441             self._RunControl('-b', 'badbranch')
 442
 443     def testBadToolchain(self):
 444         """Test that missing toolchains are detected"""
 445         self.setupToolchains();
 446         ret_code = self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
 447         lines = terminal.GetPrintTestLines()
 448
 449         # Buildman always builds the upstream commit as well
 450         self.assertIn('Building %d commits for %d boards' %
 451                 (self._commits, len(boards)), lines[0].text)
 452         self.assertEqual(self._builder.count, self._total_builds)
 453
 454         # Only sandbox should succeed, the others don't have toolchains
 455         self.assertEqual(self._builder.fail,
 456                          self._total_builds - self._commits)
 457         self.assertEqual(ret_code, 100)
 458
 459         for commit in range(self._commits):
 460             for board in self._boards.GetList():
 461                 if board.arch != 'sandbox':
 462                   errfile = self._builder.GetErrFile(commit, board.target)
 463                   fd = open(errfile)
 464                   self.assertEqual(fd.readlines(),
 465                           ['No tool chain for %s\n' % board.arch])
 466                   fd.close()
 467
 468     def testBranch(self):
 469         """Test building a branch with all toolchains present"""
 470         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
 471         self.assertEqual(self._builder.count, self._total_builds)
 472         self.assertEqual(self._builder.fail, 0)
 473
 474     def testCount(self):
 475         """Test building a specific number of commitst"""
 476         self._RunControl('-b', TEST_BRANCH, '-c2', '-o', self._output_dir)
 477         self.assertEqual(self._builder.count, 2 * len(boards))
 478         self.assertEqual(self._builder.fail, 0)
 479         # Each board has a config, and then one make per commit
 480         self.assertEqual(self._make_calls, len(boards) * (1 + 2))
 481
 482     def testIncremental(self):
 483         """Test building a branch twice - the second time should do nothing"""
 484         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
 485
 486         # Each board has a mrproper, config, and then one make per commit
 487         self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
 488         self._make_calls = 0
 489         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
 490         self.assertEqual(self._make_calls, 0)
 491         self.assertEqual(self._builder.count, self._total_builds)
 492         self.assertEqual(self._builder.fail, 0)
 493
 494     def testForceBuild(self):
 495         """The -f flag should force a rebuild"""
 496         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
 497         self._make_calls = 0
 498         self._RunControl('-b', TEST_BRANCH, '-f', '-o', self._output_dir, clean_dir=False)
 499         # Each board has a config and one make per commit
 500         self.assertEqual(self._make_calls, len(boards) * (self._commits + 1))
 501
 502     def testForceReconfigure(self):
 503         """The -f flag should force a rebuild"""
 504         self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
 505         # Each commit has a config and make
 506         self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
 507
 508     def testForceReconfigure(self):
 509         """The -f flag should force a rebuild"""
 510         self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir)
 511         # Each commit has a config and make
 512         self.assertEqual(self._make_calls, len(boards) * self._commits * 2)
 513
 514     def testMrproper(self):
 515         """The -f flag should force a rebuild"""
 516         self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir)
 517         # Each board has a mkproper, config and then one make per commit
 518         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 519
 520     def testErrors(self):
 521         """Test handling of build errors"""
 522         self._error['board2', 1] = 'fred\n'
 523         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir)
 524         self.assertEqual(self._builder.count, self._total_builds)
 525         self.assertEqual(self._builder.fail, 1)
 526
 527         # Remove the error. This should have no effect since the commit will
 528         # not be rebuilt
 529         del self._error['board2', 1]
 530         self._make_calls = 0
 531         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, clean_dir=False)
 532         self.assertEqual(self._builder.count, self._total_builds)
 533         self.assertEqual(self._make_calls, 0)
 534         self.assertEqual(self._builder.fail, 1)
 535
 536         # Now use the -F flag to force rebuild of the bad commit
 537         self._RunControl('-b', TEST_BRANCH, '-o', self._output_dir, '-F', clean_dir=False)
 538         self.assertEqual(self._builder.count, self._total_builds)
 539         self.assertEqual(self._builder.fail, 0)
 540         self.assertEqual(self._make_calls, 2)
 541
 542     def testBranchWithSlash(self):
 543         """Test building a branch with a '/' in the name"""
 544         self._test_branch = '/__dev/__testbranch'
 545         self._RunControl('-b', self._test_branch, clean_dir=False)
 546         self.assertEqual(self._builder.count, self._total_builds)
 547         self.assertEqual(self._builder.fail, 0)
 548
 549     def testWorkInOutput(self):
 550         """Test the -w option which should write directly to the output dir"""
 551         board_list = board.Boards()
 552         board_list.AddBoard(board.Board(*boards[0]))
 553         self._RunControl('-o', self._output_dir, '-w', clean_dir=False,
 554                          boards=board_list)
 555         self.assertTrue(
 556             os.path.exists(os.path.join(self._output_dir, 'u-boot')))
 557
 558     def testWorkInOutputFail(self):
 559         """Test the -w option failures"""
 560         with self.assertRaises(SystemExit) as e:
 561             self._RunControl('-o', self._output_dir, '-w', clean_dir=False)
 562         self.assertIn("single board", str(e.exception))
 563         self.assertFalse(
 564             os.path.exists(os.path.join(self._output_dir, 'u-boot')))
 565
 566         board_list = board.Boards()
 567         board_list.AddBoard(board.Board(*boards[0]))
 568         with self.assertRaises(SystemExit) as e:
 569             self._RunControl('-b', self._test_branch, '-o', self._output_dir,
 570                              '-w', clean_dir=False, boards=board_list)
 571         self.assertIn("single commit", str(e.exception))
 572
 573         board_list = board.Boards()
 574         board_list.AddBoard(board.Board(*boards[0]))
 575         with self.assertRaises(SystemExit) as e:
 576             self._RunControl('-w', clean_dir=False)
 577         self.assertIn("specify -o", str(e.exception))