scripts/oe-selftest

   1 #!/usr/bin/env python3
   2
   3 # Copyright (c) 2013 Intel Corporation
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License version 2 as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License along
  15 # with this program; if not, write to the Free Software Foundation, Inc.,
  16 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  17
  18 # DESCRIPTION
  19 # This script runs tests defined in meta/lib/oeqa/selftest/
  20 # It's purpose is to automate the testing of different bitbake tools.
  21 # To use it you just need to source your build environment setup script and
  22 # add the meta-selftest layer to your BBLAYERS.
  23 # Call the script as: "oe-selftest -a" to run all the tests in meta/lib/oeqa/selftest/
  24 # Call the script as: "oe-selftest -r <module>.<Class>.<method>" to run just a single test
  25 # E.g: "oe-selftest -r bblayers.BitbakeLayers" will run just the BitbakeLayers class from meta/lib/oeqa/selftest/bblayers.py
  26
  27
  28 import os
  29 import sys
  30 import unittest
  31 import logging
  32 import argparse
  33 import subprocess
  34 import time as t
  35 import re
  36 import fnmatch
  37 import collections
  38 import imp
  39
  40 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
  41 import scriptpath
  42 scriptpath.add_bitbake_lib_path()
  43 scriptpath.add_oe_lib_path()
  44 import argparse_oe
  45
  46 import oeqa.selftest
  47 import oeqa.utils.ftools as ftools
  48 from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
  49 from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
  50 from oeqa.selftest.base import oeSelfTest, get_available_machines
  51
  52 try:
  53     import xmlrunner
  54     from xmlrunner.result import _XMLTestResult as TestResult
  55     from xmlrunner import XMLTestRunner as _TestRunner
  56 except ImportError:
  57     # use the base runner instead
  58     from unittest import TextTestResult as TestResult
  59     from unittest import TextTestRunner as _TestRunner
  60
  61 log_prefix = "oe-selftest-" + t.strftime("%Y%m%d-%H%M%S")
  62
  63 def logger_create():
  64     log_file = log_prefix + ".log"
  65     if os.path.lexists("oe-selftest.log"):
  66         os.remove("oe-selftest.log")
  67     os.symlink(log_file, "oe-selftest.log")
  68
  69     log = logging.getLogger("selftest")
  70     log.setLevel(logging.DEBUG)
  71
  72     fh = logging.FileHandler(filename=log_file, mode='w')
  73     fh.setLevel(logging.DEBUG)
  74
  75     ch = logging.StreamHandler(sys.stdout)
  76     ch.setLevel(logging.INFO)
  77
  78     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  79     fh.setFormatter(formatter)
  80     ch.setFormatter(formatter)
  81
  82     log.addHandler(fh)
  83     log.addHandler(ch)
  84
  85     return log
  86
  87 log = logger_create()
  88
  89 def get_args_parser():
  90     description = "Script that runs unit tests against bitbake and other Yocto related tools. The goal is to validate tools functionality and metadata integrity. Refer to https://wiki.yoctoproject.org/wiki/Oe-selftest for more information."
  91     parser = argparse_oe.ArgumentParser(description=description)
  92     group = parser.add_mutually_exclusive_group(required=True)
  93     group.add_argument('-r', '--run-tests', required=False, action='store', nargs='*', dest="run_tests", default=None, help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
  94     group.add_argument('-a', '--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')
  95     group.add_argument('-m', '--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
  96     group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
  97     parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
  98     parser.add_argument('--coverage-source', dest="coverage_source", nargs="+", help="Specifiy the directories to take coverage from")
  99     parser.add_argument('--coverage-include', dest="coverage_include", nargs="+", help="Specify extra patterns to include into the coverage measurement")
 100     parser.add_argument('--coverage-omit', dest="coverage_omit", nargs="+", help="Specify with extra patterns to exclude from the coverage measurement")
 101     group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
 102                        help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
 103     group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
 104                        help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
 105     group.add_argument('-l', '--list-tests', required=False,  action="store_true", dest="list_tests", default=False,
 106                        help='List all available tests.')
 107     group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
 108                        help='List all tags that have been set to test cases.')
 109     parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
 110                         help='Run tests on different machines (random/all).')
 111     parser.add_argument('--repository', required=False, dest='repository', default='', action='store',
 112                         help='Submit test results to a repository')
 113     return parser
 114
 115 builddir = None
 116
 117
 118 def preflight_check():
 119
 120     global builddir
 121
 122     log.info("Checking that everything is in order before running the tests")
 123
 124     if not os.environ.get("BUILDDIR"):
 125         log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?")
 126         return False
 127
 128     builddir = os.environ.get("BUILDDIR")
 129     if os.getcwd() != builddir:
 130         log.info("Changing cwd to %s" % builddir)
 131         os.chdir(builddir)
 132
 133     if not "meta-selftest" in get_bb_var("BBLAYERS"):
 134         log.warn("meta-selftest layer not found in BBLAYERS, adding it")
 135         meta_selftestdir = os.path.join(
 136                 get_bb_var("BBLAYERS_FETCH_DIR"),
 137                 'meta-selftest')
 138         if os.path.isdir(meta_selftestdir):
 139             runCmd("bitbake-layers add-layer %s" %meta_selftestdir)
 140         else:
 141             log.error("could not locate meta-selftest in:\n%s"
 142                     %meta_selftestdir)
 143             return False
 144
 145     if "buildhistory.bbclass" in get_bb_var("BBINCLUDED"):
 146         log.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.")
 147         return False
 148
 149     log.info("Running bitbake -p")
 150     runCmd("bitbake -p")
 151
 152     return True
 153
 154 def add_include():
 155     global builddir
 156     if "#include added by oe-selftest.py" \
 157         not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
 158             log.info("Adding: \"include selftest.inc\" in local.conf")
 159             ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
 160                     "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
 161
 162     if "#include added by oe-selftest.py" \
 163         not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
 164             log.info("Adding: \"include bblayers.inc\" in bblayers.conf")
 165             ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \
 166                     "\n#include added by oe-selftest.py\ninclude bblayers.inc")
 167
 168 def remove_include():
 169     global builddir
 170     if builddir is None:
 171         return
 172     if "#include added by oe-selftest.py" \
 173         in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
 174             log.info("Removing the include from local.conf")
 175             ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
 176                     "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
 177
 178     if "#include added by oe-selftest.py" \
 179         in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
 180             log.info("Removing the include from bblayers.conf")
 181             ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
 182                     "\n#include added by oe-selftest.py\ninclude bblayers.inc")
 183
 184 def remove_inc_files():
 185     global builddir
 186     if builddir is None:
 187         return
 188     try:
 189         os.remove(os.path.join(builddir, "conf/selftest.inc"))
 190         for root, _, files in os.walk(get_test_layer()):
 191             for f in files:
 192                 if f == 'test_recipe.inc':
 193                     os.remove(os.path.join(root, f))
 194     except OSError as e:
 195         pass
 196
 197     for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
 198         try:
 199             os.remove(os.path.join(builddir, incl_file))
 200         except:
 201             pass
 202
 203
 204 def get_tests_modules(include_hidden=False):
 205     modules_list = list()
 206     for modules_path in oeqa.selftest.__path__:
 207         for (p, d, f) in os.walk(modules_path):
 208             files = sorted([f for f in os.listdir(p) if f.endswith('.py') and not (f.startswith('_') and not include_hidden) and not f.startswith('__') and f != 'base.py'])
 209             for f in files:
 210                 submodules = p.split("selftest")[-1]
 211                 module = ""
 212                 if submodules:
 213                     module = 'oeqa.selftest' + submodules.replace("/",".") + "." + f.split('.py')[0]
 214                 else:
 215                     module = 'oeqa.selftest.' + f.split('.py')[0]
 216                 if module not in modules_list:
 217                     modules_list.append(module)
 218     return modules_list
 219
 220
 221 def get_tests(exclusive_modules=[], include_hidden=False):
 222     test_modules = list()
 223     for x in exclusive_modules:
 224         test_modules.append('oeqa.selftest.' + x)
 225     if not test_modules:
 226         inc_hidden = include_hidden
 227         test_modules = get_tests_modules(inc_hidden)
 228
 229     return test_modules
 230
 231
 232 class Tc:
 233     def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
 234         self.tcname = tcname
 235         self.tcclass = tcclass
 236         self.tcmodule = tcmodule
 237         self.tcid = tcid
 238         # A test case can have multiple tags (as tuples) otherwise str will suffice
 239         self.tctag = tctag
 240         self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
 241
 242
 243 def get_tests_from_module(tmod):
 244     tlist = []
 245     prefix = 'oeqa.selftest.'
 246
 247     try:
 248         import importlib
 249         modlib = importlib.import_module(tmod)
 250         for mod in list(vars(modlib).values()):
 251             if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
 252                 for test in dir(mod):
 253                     if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
 254                         # Get test case id and feature tag
 255                         # NOTE: if testcase decorator or feature tag not set will throw error
 256                         try:
 257                             tid = vars(mod)[test].test_case
 258                         except:
 259                             print('DEBUG: tc id missing for ' + str(test))
 260                             tid = None
 261                         try:
 262                             ttag = vars(mod)[test].tag__feature
 263                         except:
 264                             # print('DEBUG: feature tag missing for ' + str(test))
 265                             ttag = None
 266
 267                         # NOTE: for some reason lstrip() doesn't work for mod.__module__
 268                         tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
 269     except:
 270         pass
 271
 272     return tlist
 273
 274
 275 def get_all_tests():
 276     # Get all the test modules (except the hidden ones)
 277     testlist = []
 278     tests_modules = get_tests_modules()
 279     # Get all the tests from modules
 280     for tmod in sorted(tests_modules):
 281         testlist += get_tests_from_module(tmod)
 282     return testlist
 283
 284
 285 def get_testsuite_by(criteria, keyword):
 286     # Get a testsuite based on 'keyword'
 287     # criteria: name, class, module, id, tag
 288     # keyword: a list of tests, classes, modules, ids, tags
 289
 290     ts = []
 291     all_tests = get_all_tests()
 292
 293     def get_matches(values):
 294         # Get an item and return the ones that match with keyword(s)
 295         # values: the list of items (names, modules, classes...)
 296         result = []
 297         remaining = values[:]
 298         for key in keyword:
 299             found = False
 300             if key in remaining:
 301                 # Regular matching of exact item
 302                 result.append(key)
 303                 remaining.remove(key)
 304                 found = True
 305             else:
 306                 # Wildcard matching
 307                 pattern = re.compile(fnmatch.translate(r"%s" % key))
 308                 added = [x for x in remaining if pattern.match(x)]
 309                 if added:
 310                     result.extend(added)
 311                     remaining = [x for x in remaining if x not in added]
 312                     found = True
 313             if not found:
 314                 log.error("Failed to find test: %s" % key)
 315
 316         return result
 317
 318     if criteria == 'name':
 319         names = get_matches([ tc.tcname for tc in all_tests ])
 320         ts = [ tc for tc in all_tests if tc.tcname in names ]
 321
 322     elif criteria == 'class':
 323         classes = get_matches([ tc.tcclass for tc in all_tests ])
 324         ts = [ tc for tc in all_tests if tc.tcclass in classes ]
 325
 326     elif criteria == 'module':
 327         modules = get_matches([ tc.tcmodule for tc in all_tests ])
 328         ts = [ tc for tc in all_tests if tc.tcmodule in modules ]
 329
 330     elif criteria == 'id':
 331         ids = get_matches([ str(tc.tcid) for tc in all_tests ])
 332         ts = [ tc for tc in all_tests if str(tc.tcid) in ids ]
 333
 334     elif criteria == 'tag':
 335         values = set()
 336         for tc in all_tests:
 337             # tc can have multiple tags (as tuple) otherwise str will suffice
 338             if isinstance(tc.tctag, tuple):
 339                 values |= { str(tag) for tag in tc.tctag }
 340             else:
 341                 values.add(str(tc.tctag))
 342
 343         tags = get_matches(list(values))
 344
 345         for tc in all_tests:
 346             for tag in tags:
 347                 if isinstance(tc.tctag, tuple) and tag in tc.tctag:
 348                     ts.append(tc)
 349                 elif tag == tc.tctag:
 350                     ts.append(tc)
 351
 352         # Remove duplicates from the list
 353         ts = list(set(ts))
 354
 355     return ts
 356
 357
 358 def list_testsuite_by(criteria, keyword):
 359     # Get a testsuite based on 'keyword'
 360     # criteria: name, class, module, id, tag
 361     # keyword: a list of tests, classes, modules, ids, tags
 362     def tc_key(t):
 363         if t[0] is None:
 364             return  (0,) + t[1:]
 365         return t
 366     # tcid may be None if no ID was assigned, in which case sorted() will throw
 367     # a TypeError as Python 3 does not allow comparison (<,<=,>=,>) of
 368     # heterogeneous types, handle this by using a custom key generator
 369     ts = sorted([ (tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule) \
 370                   for tc in get_testsuite_by(criteria, keyword) ], key=tc_key)
 371     print('_' * 150)
 372     for t in ts:
 373         if isinstance(t[1], (tuple, list)):
 374             print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4]))
 375         else:
 376             print('%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t)
 377     print('_' * 150)
 378     print('Filtering by:\t %s' % criteria)
 379     print('Looking for:\t %s' % ', '.join(str(x) for x in keyword))
 380     print('Total found:\t %s' % len(ts))
 381
 382
 383 def list_tests():
 384     # List all available oe-selftest tests
 385
 386     ts = get_all_tests()
 387
 388     print('%-4s\t%-10s\t%-50s' % ('id', 'tag', 'test'))
 389     print('_' * 80)
 390     for t in ts:
 391         if isinstance(t.tctag, (tuple, list)):
 392             print('%-4s\t%-10s\t%-50s' % (t.tcid, ', '.join(t.tctag), '.'.join([t.tcmodule, t.tcclass, t.tcname])))
 393         else:
 394             print('%-4s\t%-10s\t%-50s' % (t.tcid, t.tctag, '.'.join([t.tcmodule, t.tcclass, t.tcname])))
 395     print('_' * 80)
 396     print('Total found:\t %s' % len(ts))
 397
 398 def list_tags():
 399     # Get all tags set to test cases
 400     # This is useful when setting tags to test cases
 401     # The list of tags should be kept as minimal as possible
 402     tags = set()
 403     all_tests = get_all_tests()
 404
 405     for tc in all_tests:
 406         if isinstance(tc.tctag, (tuple, list)):
 407             tags.update(set(tc.tctag))
 408         else:
 409             tags.add(tc.tctag)
 410
 411     print('Tags:\t%s' % ', '.join(str(x) for x in tags))
 412
 413 def coverage_setup(coverage_source, coverage_include, coverage_omit):
 414     """ Set up the coverage measurement for the testcases to be run """
 415     import datetime
 416     import subprocess
 417     global builddir
 418     pokydir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 419     curcommit= subprocess.check_output(["git", "--git-dir", os.path.join(pokydir, ".git"), "rev-parse", "HEAD"]).decode('utf-8')
 420     coveragerc = "%s/.coveragerc" % builddir
 421     data_file = "%s/.coverage." % builddir
 422     data_file += datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
 423     if os.path.isfile(data_file):
 424         os.remove(data_file)
 425     with open(coveragerc, 'w') as cps:
 426         cps.write("# Generated with command '%s'\n" % " ".join(sys.argv))
 427         cps.write("# HEAD commit %s\n" % curcommit.strip())
 428         cps.write("[run]\n")
 429         cps.write("data_file = %s\n" % data_file)
 430         cps.write("branch = True\n")
 431         # Measure just BBLAYERS, scripts and bitbake folders
 432         cps.write("source = \n")
 433         if coverage_source:
 434             for directory in coverage_source:
 435                 if not os.path.isdir(directory):
 436                     log.warn("Directory %s is not valid.", directory)
 437                 cps.write("    %s\n" % directory)
 438         else:
 439             for layer in get_bb_var('BBLAYERS').split():
 440                 cps.write("    %s\n" % layer)
 441             cps.write("    %s\n" % os.path.dirname(os.path.realpath(__file__)))
 442             cps.write("    %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
 443
 444         if coverage_include:
 445             cps.write("include = \n")
 446             for pattern in coverage_include:
 447                 cps.write("    %s\n" % pattern)
 448         if coverage_omit:
 449             cps.write("omit = \n")
 450             for pattern in coverage_omit:
 451                 cps.write("    %s\n" % pattern)
 452
 453         return coveragerc
 454
 455 def coverage_report():
 456     """ Loads the coverage data gathered and reports it back """
 457     try:
 458         # Coverage4 uses coverage.Coverage
 459         from coverage import Coverage
 460     except:
 461         # Coverage under version 4 uses coverage.coverage
 462         from coverage import coverage as Coverage
 463
 464     import io as StringIO
 465     from coverage.misc import CoverageException
 466
 467     cov_output = StringIO.StringIO()
 468     # Creating the coverage data with the setting from the configuration file
 469     cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START'))
 470     try:
 471         # Load data from the data file specified in the configuration
 472         cov.load()
 473         # Store report data in a StringIO variable
 474         cov.report(file = cov_output, show_missing=False)
 475         log.info("\n%s" % cov_output.getvalue())
 476     except CoverageException as e:
 477         # Show problems with the reporting. Since Coverage4 not finding  any data to report raises an exception
 478         log.warn("%s" % str(e))
 479     finally:
 480         cov_output.close()
 481
 482
 483 def main():
 484     parser = get_args_parser()
 485     args = parser.parse_args()
 486
 487     # Add <layer>/lib to sys.path, so layers can add selftests
 488     log.info("Running bitbake -e to get BBPATH")
 489     bbpath = get_bb_var('BBPATH').split(':')
 490     layer_libdirs = [p for p in (os.path.join(l, 'lib') for l in bbpath) if os.path.exists(p)]
 491     sys.path.extend(layer_libdirs)
 492     imp.reload(oeqa.selftest)
 493
 494     # act like bitbake and enforce en_US.UTF-8 locale
 495     os.environ["LC_ALL"] = "en_US.UTF-8"
 496
 497     if args.run_tests_by and len(args.run_tests_by) >= 2:
 498         valid_options = ['name', 'class', 'module', 'id', 'tag']
 499         if args.run_tests_by[0] not in valid_options:
 500             print('--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0])
 501             return 1
 502         else:
 503             criteria = args.run_tests_by[0]
 504             keyword = args.run_tests_by[1:]
 505             ts = sorted([ tc.fullpath for tc in get_testsuite_by(criteria, keyword) ])
 506         if not ts:
 507             return 1
 508
 509     if args.list_tests_by and len(args.list_tests_by) >= 2:
 510         valid_options = ['name', 'class', 'module', 'id', 'tag']
 511         if args.list_tests_by[0] not in valid_options:
 512             print('--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0])
 513             return 1
 514         else:
 515             criteria = args.list_tests_by[0]
 516             keyword = args.list_tests_by[1:]
 517             list_testsuite_by(criteria, keyword)
 518
 519     if args.list_tests:
 520         list_tests()
 521
 522     if args.list_tags:
 523         list_tags()
 524
 525     if args.list_allclasses:
 526         args.list_modules = True
 527
 528     if args.list_modules:
 529         log.info('Listing all available test modules:')
 530         testslist = get_tests(include_hidden=True)
 531         for test in testslist:
 532             module = test.split('oeqa.selftest.')[-1]
 533             info = ''
 534             if module.startswith('_'):
 535                 info = ' (hidden)'
 536             print(module + info)
 537             if args.list_allclasses:
 538                 try:
 539                     import importlib
 540                     modlib = importlib.import_module(test)
 541                     for v in vars(modlib):
 542                         t = vars(modlib)[v]
 543                         if isinstance(t, type(oeSelfTest)) and issubclass(t, oeSelfTest) and t!=oeSelfTest:
 544                             print(" --", v)
 545                             for method in dir(t):
 546                                 if method.startswith("test_") and isinstance(vars(t)[method], collections.Callable):
 547                                     print(" --  --", method)
 548
 549                 except (AttributeError, ImportError) as e:
 550                     print(e)
 551                     pass
 552
 553     if args.run_tests or args.run_all_tests or args.run_tests_by:
 554         if not preflight_check():
 555             return 1
 556
 557         if args.run_tests_by:
 558             testslist = ts
 559         else:
 560             testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
 561
 562         suite = unittest.TestSuite()
 563         loader = unittest.TestLoader()
 564         loader.sortTestMethodsUsing = None
 565         runner = TestRunner(verbosity=2,
 566                 resultclass=buildResultClass(args))
 567         # we need to do this here, otherwise just loading the tests
 568         # will take 2 minutes (bitbake -e calls)
 569         oeSelfTest.testlayer_path = get_test_layer()
 570         for test in testslist:
 571             log.info("Loading tests from: %s" % test)
 572             try:
 573                 suite.addTests(loader.loadTestsFromName(test))
 574             except AttributeError as e:
 575                 log.error("Failed to import %s" % test)
 576                 log.error(e)
 577                 return 1
 578         add_include()
 579
 580         if args.machine:
 581             # Custom machine sets only weak default values (??=) for MACHINE in machine.inc
 582             # This let test cases that require a specific MACHINE to be able to override it, using (?= or =)
 583             log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine)
 584             if args.machine == 'random':
 585                 os.environ['CUSTOMMACHINE'] = 'random'
 586                 result = runner.run(suite)
 587             else:  # all
 588                 machines = get_available_machines()
 589                 for m in machines:
 590                     log.info('Run tests with custom MACHINE set to: %s' % m)
 591                     os.environ['CUSTOMMACHINE'] = m
 592                     result = runner.run(suite)
 593         else:
 594             result = runner.run(suite)
 595
 596         log.info("Finished")
 597
 598         if args.repository:
 599             import git
 600             # Commit tests results to repository
 601             metadata = metadata_from_bb()
 602             git_dir = os.path.join(os.getcwd(), 'selftest')
 603             if not os.path.isdir(git_dir):
 604                 os.mkdir(git_dir)
 605
 606             log.debug('Checking for git repository in %s' % git_dir)
 607             try:
 608                 repo = git.Repo(git_dir)
 609             except git.exc.InvalidGitRepositoryError:
 610                 log.debug("Couldn't find git repository %s; "
 611                          "cloning from %s" % (git_dir, args.repository))
 612                 repo = git.Repo.clone_from(args.repository, git_dir)
 613
 614             r_branches = repo.git.branch(r=True)
 615             r_branches = set(r_branches.replace('origin/', '').split())
 616             l_branches = {str(branch) for branch in repo.branches}
 617             branch = '%s/%s/%s' % (metadata['hostname'],
 618                                    metadata['layers']['meta'].get('branch', '(nogit)'),
 619                                    metadata['config']['MACHINE'])
 620
 621             if branch in l_branches:
 622                 log.debug('Found branch in local repository, checking out')
 623                 repo.git.checkout(branch)
 624             elif branch in r_branches:
 625                 log.debug('Found branch in remote repository, checking'
 626                           ' out and pulling')
 627                 repo.git.checkout(branch)
 628                 repo.git.pull()
 629             else:
 630                 log.debug('New branch %s' % branch)
 631                 repo.git.checkout('master')
 632                 repo.git.checkout(b=branch)
 633
 634             cleanResultsDir(repo)
 635             xml_dir = os.path.join(os.getcwd(), log_prefix)
 636             copyResultFiles(xml_dir, git_dir, repo)
 637             metadata_file = os.path.join(git_dir, 'metadata.xml')
 638             write_metadata_file(metadata_file, metadata)
 639             repo.index.add([metadata_file])
 640             repo.index.write()
 641
 642             # Get information for commit message
 643             layer_info = ''
 644             for layer, values in metadata['layers'].items():
 645                 layer_info = '%s%-17s = %s:%s\n' % (layer_info, layer,
 646                               values.get('branch', '(nogit)'), values.get('commit', '0'*40))
 647             msg = 'Selftest for build %s of %s for machine %s on %s\n\n%s' % (
 648                    log_prefix[12:], metadata['distro']['pretty_name'],
 649                    metadata['config']['MACHINE'], metadata['hostname'], layer_info)
 650
 651             log.debug('Commiting results to local repository')
 652             repo.index.commit(msg)
 653             if not repo.is_dirty():
 654                 try:
 655                     if branch in r_branches:
 656                         log.debug('Pushing changes to remote repository')
 657                         repo.git.push()
 658                     else:
 659                         log.debug('Pushing changes to remote repository '
 660                                   'creating new branch')
 661                         repo.git.push('-u', 'origin', branch)
 662                 except GitCommandError:
 663                     log.error('Falied to push to remote repository')
 664                     return 1
 665             else:
 666                 log.error('Local repository is dirty, not pushing commits')
 667
 668         if result.wasSuccessful():
 669             return 0
 670         else:
 671             return 1
 672
 673 def buildResultClass(args):
 674     """Build a Result Class to use in the testcase execution"""
 675     import site
 676
 677     class StampedResult(TestResult):
 678         """
 679         Custom TestResult that prints the time when a test starts.  As oe-selftest
 680         can take a long time (ie a few hours) to run, timestamps help us understand
 681         what tests are taking a long time to execute.
 682         If coverage is required, this class executes the coverage setup and reporting.
 683         """
 684         def startTest(self, test):
 685             import time
 686             self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
 687             super(StampedResult, self).startTest(test)
 688
 689         def startTestRun(self):
 690             """ Setup coverage before running any testcase """
 691
 692             # variable holding the coverage configuration file allowing subprocess to be measured
 693             self.coveragepth = None
 694
 695             # indicates the system if coverage is currently installed
 696             self.coverage_installed = True
 697
 698             if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
 699                 try:
 700                     # check if user can do coverage
 701                     import coverage
 702                 except:
 703                     log.warn("python coverage is not installed. More info on https://pypi.python.org/pypi/coverage")
 704                     self.coverage_installed = False
 705
 706                 if self.coverage_installed:
 707                     log.info("Coverage is enabled")
 708
 709                     major_version = int(coverage.version.__version__[0])
 710                     if major_version < 4:
 711                         log.error("python coverage %s installed. Require version 4 or greater." % coverage.version.__version__)
 712                         self.stop()
 713                     # In case the user has not set the variable COVERAGE_PROCESS_START,
 714                     # create a default one and export it. The COVERAGE_PROCESS_START
 715                     # value indicates where the coverage configuration file resides
 716                     # More info on https://pypi.python.org/pypi/coverage
 717                     if not os.environ.get('COVERAGE_PROCESS_START'):
 718                         os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.coverage_source, args.coverage_include, args.coverage_omit)
 719
 720                     # Use default site.USER_SITE and write corresponding config file
 721                     site.ENABLE_USER_SITE = True
 722                     if not os.path.exists(site.USER_SITE):
 723                         os.makedirs(site.USER_SITE)
 724                     self.coveragepth = os.path.join(site.USER_SITE, "coverage.pth")
 725                     with open(self.coveragepth, 'w') as cps:
 726                         cps.write('import sys,site; sys.path.extend(site.getsitepackages()); import coverage; coverage.process_startup();')
 727
 728         def stopTestRun(self):
 729             """ Report coverage data after the testcases are run """
 730
 731             if args.coverage or args.coverage_source or args.coverage_include or args.coverage_omit:
 732                 if self.coverage_installed:
 733                     with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
 734                         log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START'))
 735                         log.info("===========================")
 736                         log.info("\n%s" % "".join(ccf.readlines()))
 737
 738                     log.info("Coverage Report")
 739                     log.info("===============")
 740                     try:
 741                         coverage_report()
 742                     finally:
 743                         # remove the pth file
 744                         try:
 745                             os.remove(self.coveragepth)
 746                         except OSError:
 747                             log.warn("Expected temporal file from coverage is missing, ignoring removal.")
 748
 749     return StampedResult
 750
 751 def cleanResultsDir(repo):
 752     """ Remove result files from directory """
 753
 754     xml_files = []
 755     directory = repo.working_tree_dir
 756     for f in os.listdir(directory):
 757         path = os.path.join(directory, f)
 758         if os.path.isfile(path) and path.endswith('.xml'):
 759             xml_files.append(f)
 760     repo.index.remove(xml_files, working_tree=True)
 761
 762 def copyResultFiles(src, dst, repo):
 763     """ Copy result files from src to dst removing the time stamp. """
 764
 765     import shutil
 766
 767     re_time = re.compile("-[0-9]+")
 768     file_list = []
 769
 770     for root, subdirs, files in os.walk(src):
 771         tmp_dir = root.replace(src, '').lstrip('/')
 772         for s in subdirs:
 773             os.mkdir(os.path.join(dst, tmp_dir, s))
 774         for f in files:
 775             file_name = os.path.join(dst, tmp_dir, re_time.sub("", f))
 776             shutil.copy2(os.path.join(root, f), file_name)
 777             file_list.append(file_name)
 778     repo.index.add(file_list)
 779
 780 class TestRunner(_TestRunner):
 781     """Test runner class aware of exporting tests."""
 782     def __init__(self, *args, **kwargs):
 783         try:
 784             exportdir = os.path.join(os.getcwd(), log_prefix)
 785             kwargsx = dict(**kwargs)
 786             # argument specific to XMLTestRunner, if adding a new runner then
 787             # also add logic to use other runner's args.
 788             kwargsx['output'] = exportdir
 789             kwargsx['descriptions'] = False
 790             # done for the case where telling the runner where to export
 791             super(TestRunner, self).__init__(*args, **kwargsx)
 792         except TypeError:
 793             log.info("test runner init'ed like unittest")
 794             super(TestRunner, self).__init__(*args, **kwargs)
 795
 796 if __name__ == "__main__":
 797     try:
 798         ret = main()
 799     except Exception:
 800         ret = 1
 801         import traceback
 802         traceback.print_exc()
 803     finally:
 804         remove_include()
 805         remove_inc_files()
 806     sys.exit(ret)