]>
Commit | Line | Data |
---|---|---|
432496cd | 1 | #!/usr/bin/env python3 |
a87e123a JM |
2 | # |
3 | # Parallel VM test case executor | |
4aaddecd | 4 | # Copyright (c) 2014-2019, Jouni Malinen <j@w1.fi> |
a87e123a JM |
5 | # |
6 | # This software may be distributed under the terms of the BSD license. | |
7 | # See README for more details. | |
8 | ||
89896c00 | 9 | from __future__ import print_function |
71c41d45 | 10 | import curses |
a87e123a | 11 | import fcntl |
d3fb9c14 | 12 | import logging |
a87e123a | 13 | import os |
c64b6f62 | 14 | import selectors |
a87e123a JM |
15 | import subprocess |
16 | import sys | |
17 | import time | |
51c83edf | 18 | import errno |
a87e123a | 19 | |
d3fb9c14 JM |
20 | logger = logging.getLogger() |
21 | ||
68baa82c | 22 | # Test cases that take significantly longer time to execute than average. |
fab49f61 JM |
23 | long_tests = ["ap_roam_open", |
24 | "wpas_mesh_password_mismatch_retry", | |
25 | "wpas_mesh_password_mismatch", | |
26 | "hostapd_oom_wpa2_psk_connect", | |
27 | "ap_hs20_fetch_osu_stop", | |
28 | "ap_roam_wpa2_psk", | |
29 | "ibss_wpa_none_ccmp", | |
30 | "nfc_wps_er_handover_pk_hash_mismatch_sta", | |
31 | "go_neg_peers_force_diff_freq", | |
32 | "p2p_cli_invite", | |
33 | "sta_ap_scan_2b", | |
34 | "ap_pmf_sta_unprot_deauth_burst", | |
35 | "ap_bss_add_remove_during_ht_scan", | |
36 | "wext_scan_hidden", | |
37 | "autoscan_exponential", | |
38 | "nfc_p2p_client", | |
39 | "wnm_bss_keep_alive", | |
40 | "ap_inactivity_disconnect", | |
41 | "scan_bss_expiration_age", | |
42 | "autoscan_periodic", | |
43 | "discovery_group_client", | |
44 | "concurrent_p2pcli", | |
45 | "ap_bss_add_remove", | |
46 | "wpas_ap_wps", | |
47 | "wext_pmksa_cache", | |
48 | "ibss_wpa_none", | |
49 | "ap_ht_40mhz_intolerant_ap", | |
50 | "ibss_rsn", | |
51 | "discovery_pd_retries", | |
52 | "ap_wps_setup_locked_timeout", | |
53 | "ap_vht160", | |
0a809529 PKC |
54 | 'he160', |
55 | 'he160b', | |
fab49f61 JM |
56 | "dfs_radar", |
57 | "dfs", | |
58 | "dfs_ht40_minus", | |
59 | "dfs_etsi", | |
60 | "ap_acs_dfs", | |
61 | "grpform_cred_ready_timeout", | |
62 | "hostapd_oom_wpa2_eap_connect", | |
63 | "wpas_ap_dfs", | |
64 | "autogo_many", | |
65 | "hostapd_oom_wpa2_eap", | |
66 | "ibss_open", | |
67 | "proxyarp_open_ebtables", | |
68 | "proxyarp_open_ebtables_ipv6", | |
69 | "radius_failover", | |
70 | "obss_scan_40_intolerant", | |
71 | "dbus_connect_oom", | |
72 | "proxyarp_open", | |
73 | "proxyarp_open_ipv6", | |
74 | "ap_wps_iteration", | |
75 | "ap_wps_iteration_error", | |
76 | "ap_wps_pbc_timeout", | |
77 | "ap_wps_http_timeout", | |
78 | "p2p_go_move_reg_change", | |
79 | "p2p_go_move_active", | |
80 | "p2p_go_move_scm", | |
81 | "p2p_go_move_scm_peer_supports", | |
82 | "p2p_go_move_scm_peer_does_not_support", | |
83 | "p2p_go_move_scm_multi"] | |
68baa82c | 84 | |
1f4de34e | 85 | def get_failed(vm): |
71c41d45 | 86 | failed = [] |
1f4de34e JM |
87 | for i in range(num_servers): |
88 | failed += vm[i]['failed'] | |
89 | return failed | |
80411028 | 90 | |
c64b6f62 | 91 | def vm_read_stdout(vm, test_queue): |
1f4de34e | 92 | global total_started, total_passed, total_failed, total_skipped |
802bf824 | 93 | global rerun_failures |
4aaddecd | 94 | global first_run_failures |
1f4de34e | 95 | |
80411028 JM |
96 | ready = False |
97 | try: | |
98 | out = vm['proc'].stdout.read() | |
0da6d93d MH |
99 | if out == None: |
100 | return False | |
689a9560 | 101 | out = out.decode() |
51c83edf MH |
102 | except IOError as e: |
103 | if e.errno == errno.EAGAIN: | |
104 | return False | |
105 | raise | |
c64b6f62 | 106 | logger.debug("VM[%d] stdout.read[%s]" % (vm['idx'], out)) |
80411028 JM |
107 | pending = vm['pending'] + out |
108 | lines = [] | |
109 | while True: | |
110 | pos = pending.find('\n') | |
111 | if pos < 0: | |
112 | break | |
113 | line = pending[0:pos].rstrip() | |
114 | pending = pending[(pos + 1):] | |
c64b6f62 | 115 | logger.debug("VM[%d] stdout full line[%s]" % (vm['idx'], line)) |
1f4de34e | 116 | if line.startswith("READY"): |
179279eb JM |
117 | vm['starting'] = False |
118 | vm['started'] = True | |
1f4de34e JM |
119 | ready = True |
120 | elif line.startswith("PASS"): | |
80411028 | 121 | ready = True |
1f4de34e JM |
122 | total_passed += 1 |
123 | elif line.startswith("FAIL"): | |
124 | ready = True | |
125 | total_failed += 1 | |
ce591c74 JM |
126 | vals = line.split(' ') |
127 | if len(vals) < 2: | |
c64b6f62 JM |
128 | logger.info("VM[%d] incomplete FAIL line: %s" % (vm['idx'], |
129 | line)) | |
ce591c74 JM |
130 | name = line |
131 | else: | |
132 | name = vals[1] | |
c64b6f62 | 133 | logger.debug("VM[%d] test case failed: %s" % (vm['idx'], name)) |
1f4de34e | 134 | vm['failed'].append(name) |
4aaddecd | 135 | if name != vm['current_name']: |
c64b6f62 | 136 | logger.info("VM[%d] test result mismatch: %s (expected %s)" % (vm['idx'], name, vm['current_name'])) |
4aaddecd JM |
137 | else: |
138 | count = vm['current_count'] | |
139 | if count == 0: | |
140 | first_run_failures.append(name) | |
141 | if rerun_failures and count < 1: | |
142 | logger.debug("Requeue test case %s" % name) | |
143 | test_queue.append((name, vm['current_count'] + 1)) | |
1f4de34e JM |
144 | elif line.startswith("NOT-FOUND"): |
145 | ready = True | |
146 | total_failed += 1 | |
c64b6f62 | 147 | logger.info("VM[%d] test case not found" % vm['idx']) |
1f4de34e JM |
148 | elif line.startswith("SKIP"): |
149 | ready = True | |
150 | total_skipped += 1 | |
fd0465b8 JM |
151 | elif line.startswith("REASON"): |
152 | vm['skip_reason'].append(line[7:]) | |
1f4de34e JM |
153 | elif line.startswith("START"): |
154 | total_started += 1 | |
9f622398 JM |
155 | if len(vm['failed']) == 0: |
156 | vals = line.split(' ') | |
157 | if len(vals) >= 2: | |
158 | vm['fail_seq'].append(vals[1]) | |
80411028 JM |
159 | vm['out'] += line + '\n' |
160 | lines.append(line) | |
161 | vm['pending'] = pending | |
162 | return ready | |
163 | ||
c64b6f62 JM |
164 | def start_vm(vm, sel): |
165 | logger.info("VM[%d] starting up" % (vm['idx'] + 1)) | |
179279eb JM |
166 | vm['starting'] = True |
167 | vm['proc'] = subprocess.Popen(vm['cmd'], | |
168 | stdin=subprocess.PIPE, | |
169 | stdout=subprocess.PIPE, | |
170 | stderr=subprocess.PIPE) | |
171 | vm['cmd'] = None | |
172 | for stream in [vm['proc'].stdout, vm['proc'].stderr]: | |
173 | fd = stream.fileno() | |
174 | fl = fcntl.fcntl(fd, fcntl.F_GETFL) | |
175 | fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) | |
c64b6f62 | 176 | sel.register(stream, selectors.EVENT_READ, vm) |
179279eb JM |
177 | |
178 | def num_vm_starting(): | |
179 | count = 0 | |
180 | for i in range(num_servers): | |
181 | if vm[i]['starting']: | |
182 | count += 1 | |
183 | return count | |
184 | ||
c64b6f62 JM |
185 | def vm_read_stderr(vm): |
186 | try: | |
187 | err = vm['proc'].stderr.read() | |
188 | if err != None: | |
189 | err = err.decode() | |
190 | if len(err) > 0: | |
191 | vm['err'] += err | |
192 | logger.info("VM[%d] stderr.read[%s]" % (vm['idx'], err)) | |
193 | except IOError as e: | |
194 | if e.errno != errno.EAGAIN: | |
195 | raise | |
196 | ||
197 | def vm_next_step(_vm, scr, test_queue): | |
198 | scr.move(_vm['idx'] + 1, 10) | |
199 | scr.clrtoeol() | |
200 | if not test_queue: | |
201 | _vm['proc'].stdin.write(b'\n') | |
202 | _vm['proc'].stdin.flush() | |
203 | scr.addstr("shutting down") | |
204 | logger.info("VM[%d] shutting down" % _vm['idx']) | |
205 | return | |
206 | (name, count) = test_queue.pop(0) | |
207 | _vm['current_name'] = name | |
208 | _vm['current_count'] = count | |
209 | _vm['proc'].stdin.write(name.encode() + b'\n') | |
210 | _vm['proc'].stdin.flush() | |
211 | scr.addstr(name) | |
212 | logger.debug("VM[%d] start test %s" % (_vm['idx'], name)) | |
213 | ||
214 | def check_vm_start(scr, sel, test_queue): | |
215 | running = False | |
216 | updated = False | |
217 | for i in range(num_servers): | |
218 | if not vm[i]['proc']: | |
219 | # Either not yet started or already stopped VM | |
220 | if test_queue and vm[i]['cmd'] and num_vm_starting() < 2: | |
221 | scr.move(i + 1, 10) | |
222 | scr.clrtoeol() | |
223 | scr.addstr(i + 1, 10, "starting VM") | |
224 | updated = True | |
225 | start_vm(vm[i], sel) | |
226 | else: | |
227 | continue | |
228 | ||
229 | running = True | |
230 | return running, updated | |
231 | ||
232 | def vm_terminated(_vm, scr, sel, test_queue): | |
233 | updated = False | |
234 | for stream in [_vm['proc'].stdout, _vm['proc'].stderr]: | |
235 | sel.unregister(stream) | |
236 | _vm['proc'] = None | |
237 | scr.move(_vm['idx'] + 1, 10) | |
238 | scr.clrtoeol() | |
239 | log = '{}/{}.srv.{}/console'.format(dir, timestamp, _vm['idx'] + 1) | |
240 | with open(log, 'r') as f: | |
241 | if "Kernel panic" in f.read(): | |
242 | scr.addstr("kernel panic") | |
243 | logger.info("VM[%d] kernel panic" % _vm['idx']) | |
244 | updated = True | |
245 | if test_queue: | |
246 | num_vm = 0 | |
247 | for i in range(num_servers): | |
248 | if _vm['proc']: | |
249 | num_vm += 1 | |
250 | if len(test_queue) > num_vm: | |
251 | scr.addstr("unexpected exit") | |
252 | logger.info("VM[%d] unexpected exit" % i) | |
253 | updated = True | |
254 | return updated | |
255 | ||
256 | def update_screen(scr, total_tests): | |
257 | scr.move(num_servers + 1, 10) | |
258 | scr.clrtoeol() | |
259 | scr.addstr("{} %".format(int(100.0 * (total_passed + total_failed + total_skipped) / total_tests))) | |
260 | scr.addstr(num_servers + 1, 20, | |
261 | "TOTAL={} STARTED={} PASS={} FAIL={} SKIP={}".format(total_tests, total_started, total_passed, total_failed, total_skipped)) | |
262 | failed = get_failed(vm) | |
263 | if len(failed) > 0: | |
264 | scr.move(num_servers + 2, 0) | |
265 | scr.clrtoeol() | |
266 | scr.addstr("Failed test cases: ") | |
267 | count = 0 | |
268 | for f in failed: | |
269 | count += 1 | |
270 | if count > 30: | |
271 | scr.addstr('...') | |
272 | scr.clrtoeol() | |
273 | break | |
274 | scr.addstr(f) | |
275 | scr.addstr(' ') | |
276 | scr.refresh() | |
277 | ||
71c41d45 JM |
278 | def show_progress(scr): |
279 | global num_servers | |
280 | global vm | |
44ac019c JM |
281 | global dir |
282 | global timestamp | |
077f13c3 | 283 | global tests |
3eb1db03 | 284 | global first_run_failures |
1f4de34e | 285 | global total_started, total_passed, total_failed, total_skipped |
4aaddecd | 286 | global rerun_failures |
077f13c3 | 287 | |
c64b6f62 | 288 | sel = selectors.DefaultSelector() |
077f13c3 | 289 | total_tests = len(tests) |
d3fb9c14 | 290 | logger.info("Total tests: %d" % total_tests) |
4aaddecd | 291 | test_queue = [(t, 0) for t in tests] |
c64b6f62 | 292 | start_vm(vm[0], sel) |
71c41d45 JM |
293 | |
294 | scr.leaveok(1) | |
295 | scr.addstr(0, 0, "Parallel test execution status", curses.A_BOLD) | |
a87e123a | 296 | for i in range(0, num_servers): |
71c41d45 | 297 | scr.addstr(i + 1, 0, "VM %d:" % (i + 1), curses.A_BOLD) |
179279eb JM |
298 | status = "starting VM" if vm[i]['proc'] else "not yet started" |
299 | scr.addstr(i + 1, 10, status) | |
71c41d45 | 300 | scr.addstr(num_servers + 1, 0, "Total:", curses.A_BOLD) |
077f13c3 | 301 | scr.addstr(num_servers + 1, 20, "TOTAL={} STARTED=0 PASS=0 FAIL=0 SKIP=0".format(total_tests)) |
71c41d45 | 302 | scr.refresh() |
a87e123a JM |
303 | |
304 | while True: | |
a87e123a | 305 | updated = False |
c64b6f62 JM |
306 | events = sel.select(timeout=1) |
307 | for key, mask in events: | |
308 | _vm = key.data | |
309 | if not _vm['proc']: | |
a87e123a | 310 | continue |
c64b6f62 JM |
311 | vm_read_stderr(_vm) |
312 | if vm_read_stdout(_vm, test_queue): | |
313 | vm_next_step(_vm, scr, test_queue) | |
80411028 | 314 | updated = True |
c64b6f62 JM |
315 | vm_read_stderr(_vm) |
316 | if _vm['proc'].poll() is not None: | |
317 | if vm_terminated(_vm, scr, sel, test_queue): | |
318 | updated = True | |
3eb1db03 | 319 | |
c64b6f62 JM |
320 | running, run_update = check_vm_start(scr, sel, test_queue) |
321 | if updated or run_update: | |
322 | update_screen(scr, total_tests) | |
a87e123a | 323 | if not running: |
a87e123a | 324 | break |
c64b6f62 | 325 | sel.close() |
077f13c3 | 326 | |
4aaddecd JM |
327 | for i in range(num_servers): |
328 | if not vm[i]['proc']: | |
329 | continue | |
330 | vm[i]['proc'] = None | |
331 | scr.move(i + 1, 10) | |
332 | scr.clrtoeol() | |
333 | scr.addstr("still running") | |
334 | logger.info("VM[%d] still running" % i) | |
335 | ||
077f13c3 JM |
336 | scr.refresh() |
337 | time.sleep(0.3) | |
a87e123a | 338 | |
71c41d45 | 339 | def main(): |
4f06261b | 340 | import argparse |
d2002f83 | 341 | import os |
71c41d45 JM |
342 | global num_servers |
343 | global vm | |
44ac019c JM |
344 | global dir |
345 | global timestamp | |
077f13c3 | 346 | global tests |
3eb1db03 | 347 | global first_run_failures |
1f4de34e | 348 | global total_started, total_passed, total_failed, total_skipped |
802bf824 | 349 | global rerun_failures |
1f4de34e JM |
350 | |
351 | total_started = 0 | |
352 | total_passed = 0 | |
353 | total_failed = 0 | |
354 | total_skipped = 0 | |
71c41d45 | 355 | |
d3fb9c14 | 356 | debug_level = logging.INFO |
802bf824 | 357 | rerun_failures = True |
e0cccf26 JM |
358 | timestamp = int(time.time()) |
359 | ||
d2002f83 JB |
360 | scriptsdir = os.path.dirname(os.path.realpath(sys.argv[0])) |
361 | ||
4f06261b JB |
362 | p = argparse.ArgumentParser(description='run multiple testing VMs in parallel') |
363 | p.add_argument('num_servers', metavar='number of VMs', type=int, choices=range(1, 100), | |
364 | help="number of VMs to start") | |
365 | p.add_argument('-f', dest='testmodules', metavar='<test module>', | |
366 | help='execute only tests from these test modules', | |
367 | type=str, nargs='+') | |
368 | p.add_argument('-1', dest='no_retry', action='store_const', const=True, default=False, | |
369 | help="don't retry failed tests automatically") | |
370 | p.add_argument('--debug', dest='debug', action='store_const', const=True, default=False, | |
371 | help="enable debug logging") | |
372 | p.add_argument('--codecov', dest='codecov', action='store_const', const=True, default=False, | |
373 | help="enable code coverage collection") | |
374 | p.add_argument('--shuffle-tests', dest='shuffle', action='store_const', const=True, default=False, | |
375 | help="shuffle test cases to randomize order") | |
68baa82c JM |
376 | p.add_argument('--short', dest='short', action='store_const', const=True, |
377 | default=False, | |
378 | help="only run short-duration test cases") | |
4f06261b JB |
379 | p.add_argument('--long', dest='long', action='store_const', const=True, |
380 | default=False, | |
381 | help="include long-duration test cases") | |
4f76eb81 JM |
382 | p.add_argument('--valgrind', dest='valgrind', action='store_const', |
383 | const=True, default=False, | |
384 | help="run tests under valgrind") | |
e3395110 JB |
385 | p.add_argument('--telnet', dest='telnet', metavar='<baseport>', type=int, |
386 | help="enable telnet server inside VMs, specify the base port here") | |
4f06261b JB |
387 | p.add_argument('params', nargs='*') |
388 | args = p.parse_args() | |
680ce356 JB |
389 | |
390 | dir = os.environ.get('HWSIM_TEST_LOG_DIR', '/tmp/hwsim-test-logs') | |
391 | try: | |
392 | os.makedirs(dir) | |
51c83edf MH |
393 | except OSError as e: |
394 | if e.errno != errno.EEXIST: | |
395 | raise | |
680ce356 | 396 | |
4f06261b JB |
397 | num_servers = args.num_servers |
398 | rerun_failures = not args.no_retry | |
399 | if args.debug: | |
d3fb9c14 | 400 | debug_level = logging.DEBUG |
4f06261b | 401 | extra_args = [] |
4f76eb81 | 402 | if args.valgrind: |
fab49f61 | 403 | extra_args += ['--valgrind'] |
4f06261b | 404 | if args.long: |
fab49f61 | 405 | extra_args += ['--long'] |
4f06261b | 406 | if args.codecov: |
89896c00 | 407 | print("Code coverage - build separate binaries") |
680ce356 | 408 | logdir = os.path.join(dir, str(timestamp)) |
e0cccf26 | 409 | os.makedirs(logdir) |
d2002f83 JB |
410 | subprocess.check_call([os.path.join(scriptsdir, 'build-codecov.sh'), |
411 | logdir]) | |
e0cccf26 JM |
412 | codecov_args = ['--codecov_dir', logdir] |
413 | codecov = True | |
414 | else: | |
e0cccf26 JM |
415 | codecov_args = [] |
416 | codecov = False | |
417 | ||
3eb1db03 | 418 | first_run_failures = [] |
de52a2e2 JM |
419 | if args.params: |
420 | tests = args.params | |
421 | else: | |
422 | tests = [] | |
fab49f61 | 423 | cmd = [os.path.join(os.path.dirname(scriptsdir), 'run-tests.py'), '-L'] |
de52a2e2 | 424 | if args.testmodules: |
fab49f61 | 425 | cmd += ["-f"] |
de52a2e2 JM |
426 | cmd += args.testmodules |
427 | lst = subprocess.Popen(cmd, stdout=subprocess.PIPE) | |
428 | for l in lst.stdout.readlines(): | |
689a9560 | 429 | name = l.decode().split(' ')[0] |
de52a2e2 | 430 | tests.append(name) |
077f13c3 JM |
431 | if len(tests) == 0: |
432 | sys.exit("No test cases selected") | |
433 | ||
4f06261b | 434 | if args.shuffle: |
00c35673 JM |
435 | from random import shuffle |
436 | shuffle(tests) | |
437 | elif num_servers > 2 and len(tests) > 100: | |
2ffbf34d JM |
438 | # Move test cases with long duration to the beginning as an |
439 | # optimization to avoid last part of the test execution running a long | |
440 | # duration test case on a single VM while all other VMs have already | |
441 | # completed their work. | |
68baa82c | 442 | for l in long_tests: |
2ffbf34d JM |
443 | if l in tests: |
444 | tests.remove(l) | |
445 | tests.insert(0, l) | |
68baa82c JM |
446 | if args.short: |
447 | tests = [t for t in tests if t not in long_tests] | |
2ffbf34d | 448 | |
d3fb9c14 JM |
449 | logger.setLevel(debug_level) |
450 | log_handler = logging.FileHandler('parallel-vm.log') | |
451 | log_handler.setLevel(debug_level) | |
452 | fmt = "%(asctime)s %(levelname)s %(message)s" | |
453 | log_formatter = logging.Formatter(fmt) | |
454 | log_handler.setFormatter(log_formatter) | |
455 | logger.addHandler(log_handler) | |
456 | ||
71c41d45 JM |
457 | vm = {} |
458 | for i in range(0, num_servers): | |
179279eb | 459 | cmd = [os.path.join(scriptsdir, 'vm-run.sh'), |
d2002f83 | 460 | '--timestamp', str(timestamp), |
7e694225 | 461 | '--ext', 'srv.%d' % (i + 1), |
e0cccf26 | 462 | '-i'] + codecov_args + extra_args |
e3395110 | 463 | if args.telnet: |
fab49f61 | 464 | cmd += ['--telnet', str(args.telnet + i)] |
71c41d45 | 465 | vm[i] = {} |
c64b6f62 | 466 | vm[i]['idx'] = i |
179279eb JM |
467 | vm[i]['starting'] = False |
468 | vm[i]['started'] = False | |
469 | vm[i]['cmd'] = cmd | |
470 | vm[i]['proc'] = None | |
71c41d45 | 471 | vm[i]['out'] = "" |
80411028 | 472 | vm[i]['pending'] = "" |
71c41d45 | 473 | vm[i]['err'] = "" |
1f4de34e | 474 | vm[i]['failed'] = [] |
9f622398 | 475 | vm[i]['fail_seq'] = [] |
fd0465b8 | 476 | vm[i]['skip_reason'] = [] |
89896c00 | 477 | print('') |
71c41d45 JM |
478 | |
479 | curses.wrapper(show_progress) | |
480 | ||
a87e123a JM |
481 | with open('{}/{}-parallel.log'.format(dir, timestamp), 'w') as f: |
482 | for i in range(0, num_servers): | |
a2d30076 JM |
483 | f.write('VM {}\n{}\n{}\n'.format(i + 1, vm[i]['out'], vm[i]['err'])) |
484 | first = True | |
485 | for i in range(0, num_servers): | |
486 | for line in vm[i]['out'].splitlines(): | |
487 | if line.startswith("FAIL "): | |
488 | if first: | |
489 | first = False | |
490 | print("Logs for failed test cases:") | |
491 | f.write("Logs for failed test cases:\n") | |
492 | fname = "%s/%d.srv.%d/%s.log" % (dir, timestamp, i + 1, | |
493 | line.split(' ')[1]) | |
494 | print(fname) | |
495 | f.write("%s\n" % fname) | |
a87e123a | 496 | |
1f4de34e | 497 | failed = get_failed(vm) |
a87e123a | 498 | |
3eb1db03 | 499 | if first_run_failures: |
89896c00 | 500 | print("To re-run same failure sequence(s):") |
9f622398 JM |
501 | for i in range(0, num_servers): |
502 | if len(vm[i]['failed']) == 0: | |
503 | continue | |
89896c00 | 504 | print("./vm-run.sh", end=' ') |
45e8a45b | 505 | if args.long: |
89896c00 | 506 | print("--long", end=' ') |
9f622398 JM |
507 | skip = len(vm[i]['fail_seq']) |
508 | skip -= min(skip, 30) | |
509 | for t in vm[i]['fail_seq']: | |
510 | if skip > 0: | |
511 | skip -= 1 | |
512 | continue | |
89896c00 MH |
513 | print(t, end=' ') |
514 | print('') | |
515 | print("Failed test cases:") | |
3eb1db03 | 516 | for f in first_run_failures: |
89896c00 | 517 | print(f, end=' ') |
d3fb9c14 | 518 | logger.info("Failed: " + f) |
89896c00 | 519 | print('') |
3eb1db03 | 520 | double_failed = [] |
1f4de34e | 521 | for name in failed: |
3eb1db03 JM |
522 | double_failed.append(name) |
523 | for test in first_run_failures: | |
524 | double_failed.remove(test) | |
802bf824 JM |
525 | if not rerun_failures: |
526 | pass | |
527 | elif failed and not double_failed: | |
89896c00 | 528 | print("All failed cases passed on retry") |
d3fb9c14 | 529 | logger.info("All failed cases passed on retry") |
3eb1db03 | 530 | elif double_failed: |
89896c00 | 531 | print("Failed even on retry:") |
3eb1db03 | 532 | for f in double_failed: |
89896c00 | 533 | print(f, end=' ') |
d3fb9c14 | 534 | logger.info("Failed on retry: " + f) |
89896c00 | 535 | print('') |
1f4de34e JM |
536 | res = "TOTAL={} PASS={} FAIL={} SKIP={}".format(total_started, |
537 | total_passed, | |
538 | total_failed, | |
539 | total_skipped) | |
d3fb9c14 JM |
540 | print(res) |
541 | logger.info(res) | |
89896c00 | 542 | print("Logs: " + dir + '/' + str(timestamp)) |
d3fb9c14 | 543 | logger.info("Logs: " + dir + '/' + str(timestamp)) |
a87e123a | 544 | |
fd0465b8 | 545 | skip_reason = [] |
4aaddecd | 546 | for i in range(num_servers): |
179279eb JM |
547 | if not vm[i]['started']: |
548 | continue | |
fd0465b8 | 549 | skip_reason += vm[i]['skip_reason'] |
80411028 JM |
550 | if len(vm[i]['pending']) > 0: |
551 | logger.info("Unprocessed stdout from VM[%d]: '%s'" % | |
552 | (i, vm[i]['pending'])) | |
44ac019c JM |
553 | log = '{}/{}.srv.{}/console'.format(dir, timestamp, i + 1) |
554 | with open(log, 'r') as f: | |
555 | if "Kernel panic" in f.read(): | |
89896c00 | 556 | print("Kernel panic in " + log) |
d3fb9c14 | 557 | logger.info("Kernel panic in " + log) |
fd0465b8 JM |
558 | missing = {} |
559 | missing['OCV not supported'] = 'OCV' | |
560 | missing['sigma_dut not available'] = 'sigma_dut' | |
561 | missing['Skip test case with long duration due to --long not specified'] = 'long' | |
562 | missing['TEST_ALLOC_FAIL not supported' ] = 'TEST_FAIL' | |
563 | missing['TEST_ALLOC_FAIL not supported in the build'] = 'TEST_FAIL' | |
564 | missing['TEST_FAIL not supported' ] = 'TEST_FAIL' | |
565 | missing['veth not supported (kernel CONFIG_VETH)'] = 'KERNEL:CONFIG_VETH' | |
566 | missing['WPA-EAP-SUITE-B-192 not supported'] = 'CONFIG_SUITEB192' | |
567 | missing['WPA-EAP-SUITE-B not supported'] = 'CONFIG_SUITEB' | |
568 | missing['wmediumd not available'] = 'wmediumd' | |
569 | missing['DPP not supported'] = 'CONFIG_DPP' | |
570 | missing['DPP version 2 not supported'] = 'CONFIG_DPP2' | |
571 | missing_items = [] | |
572 | other_reasons = [] | |
573 | for reason in sorted(set(skip_reason)): | |
574 | if reason in missing: | |
575 | missing_items.append(missing[reason]) | |
576 | elif reason.startswith('OCSP-multi not supported with this TLS library'): | |
577 | missing_items.append('OCSP-MULTI') | |
578 | else: | |
579 | other_reasons.append(reason) | |
580 | if missing_items: | |
581 | print("Missing items (SKIP):", missing_items) | |
582 | if other_reasons: | |
583 | print("Other skip reasons:", other_reasons) | |
44ac019c | 584 | |
e0cccf26 | 585 | if codecov: |
89896c00 | 586 | print("Code coverage - preparing report") |
e0cccf26 | 587 | for i in range(num_servers): |
d2002f83 JB |
588 | subprocess.check_call([os.path.join(scriptsdir, |
589 | 'process-codecov.sh'), | |
e0cccf26 JM |
590 | logdir + ".srv.%d" % (i + 1), |
591 | str(i)]) | |
d2002f83 JB |
592 | subprocess.check_call([os.path.join(scriptsdir, 'combine-codecov.sh'), |
593 | logdir]) | |
89896c00 | 594 | print("file://%s/index.html" % logdir) |
d3fb9c14 | 595 | logger.info("Code coverage report: file://%s/index.html" % logdir) |
e0cccf26 | 596 | |
802bf824 | 597 | if double_failed or (failed and not rerun_failures): |
d3fb9c14 | 598 | logger.info("Test run complete - failures found") |
3eb1db03 JM |
599 | sys.exit(2) |
600 | if failed: | |
d3fb9c14 | 601 | logger.info("Test run complete - failures found on first run; passed on retry") |
3eb1db03 | 602 | sys.exit(1) |
d3fb9c14 | 603 | logger.info("Test run complete - no failures") |
3eb1db03 JM |
604 | sys.exit(0) |
605 | ||
a87e123a JM |
606 | if __name__ == "__main__": |
607 | main() |