[thirdparty/hostap.git] / tests / remote / run-tests.py

#!/usr/bin/env python3
#
# Remote test case executor
# Copyright (c) 2016, Tieto Corporation
#
# This software may be distributed under the terms of the BSD license.
# See README for more details.

import os
import re
import sys
import time
import traceback
import getopt
from datetime import datetime

import logging
logger = logging.getLogger()

scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
sys.path.append(os.path.join(scriptsdir, '..', 'hwsim'))

import wpaspy
import config
from test_devices import show_devices
from test_devices import check_devices
from rutils import TestSkip
from utils import HwsimSkip
from hwsim_wrapper import run_hwsim_test

def usage():
    print("USAGE: " + sys.argv[0] + " -t devices")
    print("USAGE: " + sys.argv[0] + " -t check_devices")
    print("USAGE: " + sys.argv[0] + " -d <dut_name> -t <all|sanity|tests_to_run> [-r <ref_name>] [-c <cfg_file.py>] [-m <all|monitor_name>] [-h hwsim_tests][-R][-T][-P][-v]")
    print("USAGE: " + sys.argv[0])

def get_devices(devices, duts, refs, monitors):
    for dut in duts:
        config.get_device(devices, dut, lock=True)
    for ref in refs:
        config.get_device(devices, ref, lock=True)
    for monitor in monitors:
        if monitor == "all":
            continue
        if monitor in duts:
            continue
        if monitor in refs:
            continue
        config.get_device(devices, monitor, lock=True)

def put_devices(devices, duts, refs, monitors):
    for dut in duts:
        config.put_device(devices, dut)
    for ref in refs:
        config.put_device(devices, ref)
    for monitor in monitors:
        if monitor == "all":
            continue
        if monitor in duts:
            continue
        if monitor in refs:
            continue
        config.put_device(devices, monitor)

def main():
    duts = []
    refs = []
    monitors = []
    filter_keys = []
    requested_tests = ["help"]
    requested_hwsim_tests = []
    hwsim_tests = []
    cfg_file = "cfg.py"
    log_dir = "./logs/"
    verbose = False
    trace = False
    restart = False
    perf = False

    # parse input parameters
    try:
        opts, args = getopt.getopt(sys.argv[1:], "d:r:t:l:k:c:m:h:vRPT",
                                   ["dut=", "ref=", "tests=", "log-dir=",
                                    "cfg=", "key=", "monitor=", "hwsim="])
    except getopt.GetoptError as err:
        print(err)
        usage()
        sys.exit(2)

    for option, argument in opts:
        if option == "-v":
            verbose = True
        elif option == "-R":
            restart = True
        elif option == "-T":
            trace = True
        elif option == "-P":
            perf = True
        elif option in ("-d", "--dut"):
            duts.append(argument)
        elif option in ("-r", "--ref"):
            refs.append(argument)
        elif option in ("-t", "--tests"):
            requested_tests = re.split('; | |, ', argument)
        elif option in ("-l", "--log-dir"):
            log_dir = argument
        elif option in ("-k", "--key"):
            filter_keys.append(argument)
        elif option in ("-m", "--monitor"):
            monitors.append(argument)
        elif option in ("-c", "--cfg"):
            cfg_file = argument
        elif option in ("-h", "--hwsim"):
            requested_hwsim_tests = re.split('; | |, ', argument)
        else:
            assert False, "unhandled option"

    # get env configuration
    setup_params = config.get_setup_params(cfg_file)
    devices = config.get_devices(cfg_file)

    # put logs in log_dir
    symlink = os.path.join(log_dir, "current");
    if os.path.exists(symlink):
        os.unlink(symlink)
    log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S"))
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    os.symlink(os.path.join("../", log_dir), symlink)

    # setup restart/trace/perf request
    setup_params['local_log_dir'] = log_dir
    setup_params['restart_device'] = restart
    setup_params['trace'] = trace
    setup_params['perf'] = perf

    # configure logger
    logger.setLevel(logging.DEBUG)

    stdout_handler = logging.StreamHandler()
    stdout_handler.setLevel(logging.WARNING)
    if verbose:
        stdout_handler.setLevel(logging.DEBUG)
    logger.addHandler(stdout_handler)

    formatter = logging.Formatter('%(asctime)s - %(message)s')
    file_name = os.path.join(log_dir, 'run-tests.log')
    log_handler = logging.FileHandler(file_name)
    log_handler.setLevel(logging.DEBUG)
    log_handler.setFormatter(formatter)
    logger.addHandler(log_handler)

    # import available tests
    tests = []
    failed = []
    test_modules = []
    files = os.listdir(scriptsdir)
    for t in files:
        m = re.match(r'(test_.*)\.py$', t)
        if m:
            mod = __import__(m.group(1))
            test_modules.append(mod.__name__.replace('test_', '', 1))
            for key, val in mod.__dict__.items():
                if key.startswith("test_"):
                    tests.append(val)
    test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests]))

    # import test_*
    files = os.listdir("../hwsim/")
    for t in files:
        m = re.match(r'(test_.*)\.py$', t)
        if m:
            mod = __import__(m.group(1))
            test_modules.append(mod.__name__.replace('test_', '', 1))
            for key, val in mod.__dict__.items():
                if key.startswith("test_"):
                    hwsim_tests.append(val)

    # setup hwsim tests
    hwsim_tests_to_run = []
    if len(requested_hwsim_tests) > 0:
        # apply filters
        for filter_key in filter_keys:
            filtered_tests = []
            for hwsim_test in hwsim_tests:
                if re.search(filter_key, hwsim_test.__name__):
                    filtered_tests.append(hwsim_test)
            hwsim_tests = filtered_tests

        # setup hwsim_test we should run
        if requested_hwsim_tests[0] == "all":
            hwsim_tests_to_run = hwsim_tests
        elif requested_hwsim_tests[0] == "remote":
            hwsim_tests_to_run = [t for t in hwsim_tests
                                  if hasattr(t, "remote_compatible") and
                                     t.remote_compatible]
        else:
            for test in requested_hwsim_tests:
                t = None
                for tt in hwsim_tests:
                    name = tt.__name__.replace('test_', '', 1)
                    if name == test and tt.__code__.co_argcount <= 2:
                        t = tt
                        break
                if not t:
                    logger.warning("hwsim test case: " + test + " NOT-FOUND")
                    continue
                hwsim_tests_to_run.append(t)

    # sort the list
    test_names.sort()
    tests.sort()

    # print help
    if requested_tests[0] == "help" and len(requested_hwsim_tests) == 0:
        usage()
        print("\nAvailable Devices:")
        for device in devices:
            print("\t", device['name'])
        print("\nAvailable tests:")
        for test in test_names:
            print("\t", test)
        print("\nAvailable hwsim tests:")
        for hwsim_test in hwsim_tests:
            print("\t", hwsim_test.__name__.replace('test_', '', 1))
        return

    # show/check devices
    if requested_tests[0] == "devices":
        show_devices(devices, setup_params)
        return

    # apply filters
    for filter_key in filter_keys:
        filtered_tests = []
        for test in tests:
            if re.search(filter_key, test.__name__):
                filtered_tests.append(test)
        tests = filtered_tests

    # setup test we should run
    tests_to_run = []
    if requested_tests[0] == "all":
        tests_to_run = tests
    if requested_tests[0] == "help":
        pass
    elif requested_tests[0] == "sanity":
        for test in tests:
            if test.__name__.startswith("test_sanity_"):
                tests_to_run.append(test)
    else:
        for test in requested_tests:
            t = None
            for tt in tests:
                name = tt.__name__.replace('test_', '', 1)
                if name == test:
                    t = tt
                    break
            if not t:
                logger.warning("test case: " + test + " NOT-FOUND")
                continue
            tests_to_run.append(t)

    # lock devices
    try:
        get_devices(devices, duts, refs, monitors)
    except Exception as e:
        logger.warning("get devices failed: " + str(e))
        logger.info(traceback.format_exc())
        put_devices(devices, duts, refs, monitors)
        return
    except:
        logger.warning("get devices failed")
        logger.info(traceback.format_exc())
        put_devices(devices, duts, refs, monitors)
        return

    # now run test cases
    for dut in duts:
        if len(requested_hwsim_tests) > 0:
            logger.warning("DUT (apdev): " + str(dut))
        else:
            logger.warning("DUT: " + str(dut))
    for ref in refs:
        if len(requested_hwsim_tests) > 0:
            logger.warning("REF   (dev): " + str(ref))
        else:
            logger.warning("REF: " + str(ref))
    for monitor in monitors:
        logger.warning("MON: " + str(monitor))

    # run check_devices at begining
    logger.warning("RUN check_devices")
    try:
        check_devices(devices, setup_params, refs, duts, monitors)
    except Exception as e:
        logger.warning("FAILED: " + str(e))
        logger.info(traceback.format_exc())
        put_devices(devices, duts, refs, monitors)
        return
    except:
        logger.warning("FAILED")
        logger.info(traceback.format_exc())
        put_devices(devices, duts, refs, monitors)
        return
    logger.warning("PASS")

    test_no = 1
    for test in tests_to_run:
        try:
            start = datetime.now()
            setup_params['tc_name'] = test.__name__.replace('test_', '', 1)
            logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")")
            if test.__doc__:
                logger.info("Test: " + test.__doc__)

            # run tc
            res = test(devices, setup_params, refs, duts, monitors)

            end = datetime.now()
            logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
        except KeyboardInterrupt:
            put_devices(devices, duts, refs, monitors)
            raise
        except TestSkip as e:
            end = datetime.now()
            logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
        except Exception as e:
            end = datetime.now()
            logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
            logger.info(traceback.format_exc())
            failed.append(test.__name__.replace('test_', '', 1))
        except:
            end = datetime.now()
            logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
            logger.info(traceback.format_exc())
            failed.append(test.__name__.replace('test_', '', 1))
        test_no += 1

    test_no = 1
    for hwsim_test in hwsim_tests_to_run:
        try:
            start = datetime.now()
            setup_params['tc_name'] = hwsim_test.__name__.replace('test_', '', 1)
            logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(hwsim_tests_to_run)) + ")")
            res = run_hwsim_test(devices, setup_params, refs, duts, monitors, hwsim_test)
            end = datetime.now()
            logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
        except KeyboardInterrupt:
            put_devices(devices, duts, refs, monitors)
            raise
        except HwsimSkip as e:
            end = datetime.now()
            logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
            failed.append(hwsim_test.__name__.replace('test_', '', 1))
        except Exception as e:
            end = datetime.now()
            logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
            logger.info(traceback.format_exc())
            failed.append(hwsim_test.__name__.replace('test_', '', 1))
        except:
            end = datetime.now()
            logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
            logger.info(traceback.format_exc())
            failed.append(hwsim_test.__name__.replace('test_', '', 1))
        test_no += 1

    # unlock devices
    put_devices(devices, duts, refs, monitors)

    if len(failed) > 0:
        logger.warning("Failed test cases:")
        for test in failed:
            logger.warning("\t" + test)


if __name__ == "__main__":
        main()
Commit	Line	Data
ec5de195	1	#!/usr/bin/env python3
5865186e JD	2	#
	3	# Remote test case executor
	4	# Copyright (c) 2016, Tieto Corporation
	5	#
	6	# This software may be distributed under the terms of the BSD license.
	7	# See README for more details.
	8
	9	import os
	10	import re
	11	import sys
	12	import time
	13	import traceback
	14	import getopt
	15	from datetime import datetime
	16
	17	import logging
	18	logger = logging.getLogger()
	19
	20	scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
	21	sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
	22	sys.path.append(os.path.join(scriptsdir, '..', 'hwsim'))
	23
	24	import wpaspy
	25	import config
	26	from test_devices import show_devices
	27	from test_devices import check_devices
a73fa13b	28	from rutils import TestSkip
a185e9b1 JD	29	from utils import HwsimSkip
a185e9b1 JD	30	from hwsim_wrapper import run_hwsim_test
5865186e JD	31
5865186e JD	32	def usage():
89896c00 MH	33	print("USAGE: " + sys.argv[0] + " -t devices")
	34	print("USAGE: " + sys.argv[0] + " -t check_devices")
	35	print("USAGE: " + sys.argv[0] + " -d <dut_name> -t <all\|sanity\|tests_to_run> [-r <ref_name>] [-c <cfg_file.py>] [-m <all\|monitor_name>] [-h hwsim_tests][-R][-T][-P][-v]")
	36	print("USAGE: " + sys.argv[0])
5865186e JD	37
	38	def get_devices(devices, duts, refs, monitors):
	39	for dut in duts:
	40	config.get_device(devices, dut, lock=True)
	41	for ref in refs:
	42	config.get_device(devices, ref, lock=True)
	43	for monitor in monitors:
	44	if monitor == "all":
	45	continue
	46	if monitor in duts:
	47	continue
	48	if monitor in refs:
	49	continue
	50	config.get_device(devices, monitor, lock=True)
	51
	52	def put_devices(devices, duts, refs, monitors):
	53	for dut in duts:
	54	config.put_device(devices, dut)
	55	for ref in refs:
	56	config.put_device(devices, ref)
	57	for monitor in monitors:
	58	if monitor == "all":
	59	continue
	60	if monitor in duts:
	61	continue
	62	if monitor in refs:
	63	continue
	64	config.put_device(devices, monitor)
	65
	66	def main():
	67	duts = []
	68	refs = []
	69	monitors = []
	70	filter_keys = []
	71	requested_tests = ["help"]
a185e9b1 JD	72	requested_hwsim_tests = []
a185e9b1 JD	73	hwsim_tests = []
5865186e JD	74	cfg_file = "cfg.py"
	75	log_dir = "./logs/"
	76	verbose = False
	77	trace = False
	78	restart = False
	79	perf = False
	80
	81	# parse input parameters
	82	try:
a185e9b1	83	opts, args = getopt.getopt(sys.argv[1:], "d:r:t:l:k:c:m:h:vRPT",
5865186e	84	["dut=", "ref=", "tests=", "log-dir=",
a185e9b1	85	"cfg=", "key=", "monitor=", "hwsim="])
5865186e JD	86	except getopt.GetoptError as err:
	87	print(err)
	88	usage()
	89	sys.exit(2)
	90
	91	for option, argument in opts:
	92	if option == "-v":
	93	verbose = True
	94	elif option == "-R":
	95	restart = True
	96	elif option == "-T":
	97	trace = True
	98	elif option == "-P":
	99	perf = True
	100	elif option in ("-d", "--dut"):
	101	duts.append(argument)
	102	elif option in ("-r", "--ref"):
	103	refs.append(argument)
	104	elif option in ("-t", "--tests"):
	105	requested_tests = re.split('; \| \|, ', argument)
	106	elif option in ("-l", "--log-dir"):
	107	log_dir = argument
	108	elif option in ("-k", "--key"):
	109	filter_keys.append(argument)
	110	elif option in ("-m", "--monitor"):
	111	monitors.append(argument)
	112	elif option in ("-c", "--cfg"):
	113	cfg_file = argument
a185e9b1 JD	114	elif option in ("-h", "--hwsim"):
a185e9b1 JD	115	requested_hwsim_tests = re.split('; \| \|, ', argument)
5865186e JD	116	else:
	117	assert False, "unhandled option"
	118
	119	# get env configuration
	120	setup_params = config.get_setup_params(cfg_file)
	121	devices = config.get_devices(cfg_file)
	122
	123	# put logs in log_dir
	124	symlink = os.path.join(log_dir, "current");
	125	if os.path.exists(symlink):
	126	os.unlink(symlink)
	127	log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S"))
	128	if not os.path.exists(log_dir):
	129	os.makedirs(log_dir)
	130	os.symlink(os.path.join("../", log_dir), symlink)
	131
	132	# setup restart/trace/perf request
	133	setup_params['local_log_dir'] = log_dir
	134	setup_params['restart_device'] = restart
	135	setup_params['trace'] = trace
	136	setup_params['perf'] = perf
	137
	138	# configure logger
	139	logger.setLevel(logging.DEBUG)
	140
	141	stdout_handler = logging.StreamHandler()
	142	stdout_handler.setLevel(logging.WARNING)
	143	if verbose:
	144	stdout_handler.setLevel(logging.DEBUG)
	145	logger.addHandler(stdout_handler)
	146
	147	formatter = logging.Formatter('%(asctime)s - %(message)s')
	148	file_name = os.path.join(log_dir, 'run-tests.log')
	149	log_handler = logging.FileHandler(file_name)
	150	log_handler.setLevel(logging.DEBUG)
	151	log_handler.setFormatter(formatter)
	152	logger.addHandler(log_handler)
	153
	154	# import available tests
	155	tests = []
	156	failed = []
	157	test_modules = []
	158	files = os.listdir(scriptsdir)
	159	for t in files:
	160	m = re.match(r'(test_.*)\.py$', t)
	161	if m:
	162	mod = __import__(m.group(1))
	163	test_modules.append(mod.__name__.replace('test_', '', 1))
fab49f61	164	for key, val in mod.__dict__.items():
5865186e JD	165	if key.startswith("test_"):
	166	tests.append(val)
	167	test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests]))
	168
a185e9b1 JD	169	# import test_*
	170	files = os.listdir("../hwsim/")
	171	for t in files:
	172	m = re.match(r'(test_.*)\.py$', t)
	173	if m:
	174	mod = __import__(m.group(1))
	175	test_modules.append(mod.__name__.replace('test_', '', 1))
fab49f61	176	for key, val in mod.__dict__.items():
a185e9b1 JD	177	if key.startswith("test_"):
	178	hwsim_tests.append(val)
	179
	180	# setup hwsim tests
	181	hwsim_tests_to_run = []
	182	if len(requested_hwsim_tests) > 0:
	183	# apply filters
	184	for filter_key in filter_keys:
	185	filtered_tests = []
	186	for hwsim_test in hwsim_tests:
	187	if re.search(filter_key, hwsim_test.__name__):
	188	filtered_tests.append(hwsim_test)
	189	hwsim_tests = filtered_tests
	190
	191	# setup hwsim_test we should run
	192	if requested_hwsim_tests[0] == "all":
	193	hwsim_tests_to_run = hwsim_tests
5aa751ea JA	194	elif requested_hwsim_tests[0] == "remote":
	195	hwsim_tests_to_run = [t for t in hwsim_tests
	196	if hasattr(t, "remote_compatible") and
	197	t.remote_compatible]
a185e9b1 JD	198	else:
	199	for test in requested_hwsim_tests:
	200	t = None
	201	for tt in hwsim_tests:
	202	name = tt.__name__.replace('test_', '', 1)
8c851c2e	203	if name == test and tt.__code__.co_argcount <= 2:
a185e9b1 JD	204	t = tt
	205	break
	206	if not t:
	207	logger.warning("hwsim test case: " + test + " NOT-FOUND")
	208	continue
	209	hwsim_tests_to_run.append(t)
	210
5865186e JD	211	# sort the list
	212	test_names.sort()
	213	tests.sort()
	214
	215	# print help
a185e9b1	216	if requested_tests[0] == "help" and len(requested_hwsim_tests) == 0:
5865186e	217	usage()
89896c00	218	print("\nAvailable Devices:")
5865186e	219	for device in devices:
89896c00 MH	220	print("\t", device['name'])
89896c00 MH	221	print("\nAvailable tests:")
5865186e	222	for test in test_names:
89896c00 MH	223	print("\t", test)
89896c00 MH	224	print("\nAvailable hwsim tests:")
a185e9b1	225	for hwsim_test in hwsim_tests:
89896c00	226	print("\t", hwsim_test.__name__.replace('test_', '', 1))
5865186e JD	227	return
	228
	229	# show/check devices
	230	if requested_tests[0] == "devices":
	231	show_devices(devices, setup_params)
	232	return
	233
	234	# apply filters
	235	for filter_key in filter_keys:
	236	filtered_tests = []
	237	for test in tests:
	238	if re.search(filter_key, test.__name__):
	239	filtered_tests.append(test)
	240	tests = filtered_tests
	241
	242	# setup test we should run
	243	tests_to_run = []
	244	if requested_tests[0] == "all":
	245	tests_to_run = tests
a185e9b1 JD	246	if requested_tests[0] == "help":
a185e9b1 JD	247	pass
5865186e JD	248	elif requested_tests[0] == "sanity":
	249	for test in tests:
	250	if test.__name__.startswith("test_sanity_"):
	251	tests_to_run.append(test)
	252	else:
	253	for test in requested_tests:
	254	t = None
	255	for tt in tests:
	256	name = tt.__name__.replace('test_', '', 1)
	257	if name == test:
	258	t = tt
	259	break
	260	if not t:
	261	logger.warning("test case: " + test + " NOT-FOUND")
	262	continue
	263	tests_to_run.append(t)
	264
	265	# lock devices
	266	try:
	267	get_devices(devices, duts, refs, monitors)
bab493b9	268	except Exception as e:
5865186e JD	269	logger.warning("get devices failed: " + str(e))
	270	logger.info(traceback.format_exc())
	271	put_devices(devices, duts, refs, monitors)
	272	return
	273	except:
	274	logger.warning("get devices failed")
	275	logger.info(traceback.format_exc())
	276	put_devices(devices, duts, refs, monitors)
	277	return
	278
	279	# now run test cases
	280	for dut in duts:
876785ae JD	281	if len(requested_hwsim_tests) > 0:
	282	logger.warning("DUT (apdev): " + str(dut))
	283	else:
	284	logger.warning("DUT: " + str(dut))
5865186e	285	for ref in refs:
876785ae JD	286	if len(requested_hwsim_tests) > 0:
	287	logger.warning("REF (dev): " + str(ref))
	288	else:
	289	logger.warning("REF: " + str(ref))
5865186e JD	290	for monitor in monitors:
	291	logger.warning("MON: " + str(monitor))
	292
	293	# run check_devices at begining
	294	logger.warning("RUN check_devices")
	295	try:
	296	check_devices(devices, setup_params, refs, duts, monitors)
bab493b9	297	except Exception as e:
5865186e JD	298	logger.warning("FAILED: " + str(e))
	299	logger.info(traceback.format_exc())
	300	put_devices(devices, duts, refs, monitors)
	301	return
	302	except:
	303	logger.warning("FAILED")
	304	logger.info(traceback.format_exc())
	305	put_devices(devices, duts, refs, monitors)
	306	return
	307	logger.warning("PASS")
	308
	309	test_no = 1
	310	for test in tests_to_run:
	311	try:
	312	start = datetime.now()
	313	setup_params['tc_name'] = test.__name__.replace('test_', '', 1)
	314	logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")")
	315	if test.__doc__:
	316	logger.info("Test: " + test.__doc__)
	317
	318	# run tc
	319	res = test(devices, setup_params, refs, duts, monitors)
	320
	321	end = datetime.now()
	322	logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
	323	except KeyboardInterrupt:
	324	put_devices(devices, duts, refs, monitors)
	325	raise
bab493b9	326	except TestSkip as e:
a73fa13b JD	327	end = datetime.now()
a73fa13b JD	328	logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
bab493b9	329	except Exception as e:
5865186e JD	330	end = datetime.now()
	331	logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
	332	logger.info(traceback.format_exc())
	333	failed.append(test.__name__.replace('test_', '', 1))
	334	except:
	335	end = datetime.now()
	336	logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
	337	logger.info(traceback.format_exc())
	338	failed.append(test.__name__.replace('test_', '', 1))
	339	test_no += 1
	340
a185e9b1 JD	341	test_no = 1
	342	for hwsim_test in hwsim_tests_to_run:
	343	try:
	344	start = datetime.now()
	345	setup_params['tc_name'] = hwsim_test.__name__.replace('test_', '', 1)
	346	logger.warning("START - " + setup_params['tc_name'] + " (" + str(test_no) + "/" + str(len(hwsim_tests_to_run)) + ")")
	347	res = run_hwsim_test(devices, setup_params, refs, duts, monitors, hwsim_test)
	348	end = datetime.now()
	349	logger.warning("PASS (" + res + ") - " + str((end - start).total_seconds()) + "s")
	350	except KeyboardInterrupt:
	351	put_devices(devices, duts, refs, monitors)
	352	raise
bab493b9	353	except HwsimSkip as e:
a185e9b1 JD	354	end = datetime.now()
	355	logger.warning("SKIP (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
	356	failed.append(hwsim_test.__name__.replace('test_', '', 1))
bab493b9	357	except Exception as e:
a185e9b1 JD	358	end = datetime.now()
	359	logger.warning("FAILED (" + str(e) + ") - " + str((end - start).total_seconds()) + "s")
	360	logger.info(traceback.format_exc())
	361	failed.append(hwsim_test.__name__.replace('test_', '', 1))
	362	except:
	363	end = datetime.now()
	364	logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
	365	logger.info(traceback.format_exc())
	366	failed.append(hwsim_test.__name__.replace('test_', '', 1))
	367	test_no += 1
	368
5865186e JD	369	# unlock devices
	370	put_devices(devices, duts, refs, monitors)
	371
	372	if len(failed) > 0:
	373	logger.warning("Failed test cases:")
	374	for test in failed:
	375	logger.warning("\t" + test)
	376
	377
	378	if __name__ == "__main__":
	379	main()