]>
git.ipfire.org Git - thirdparty/hostap.git/blob - tests/hwsim/run-tests.py
4 # Copyright (c) 2013, Jouni Malinen <j@w1.fi>
6 # This software may be distributed under the terms of the BSD license.
7 # See README for more details.
13 from datetime
import datetime
18 logger
= logging
.getLogger()
20 sys
.path
.append('../../wpaspy')
22 from wpasupplicant
import WpaSupplicant
23 from hostapd
import HostapdGlobal
24 from check_kernel
import check_kernel
26 def reset_devs(dev
, apdev
):
32 logger
.info("Failed to reset device " + d
.ifname
)
36 hapd
= HostapdGlobal()
37 hapd
.remove('wlan3-3')
38 hapd
.remove('wlan3-2')
40 hapd
.remove(ap
['ifname'])
42 logger
.info("Failed to remove hostapd interface")
47 def report(conn
, prefill
, build
, commit
, run
, test
, result
, duration
):
54 conn
.execute('DELETE FROM results WHERE test=? AND run=? AND result=?', (test
, run
, 'NOTRUN'))
55 sql
= "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)"
56 params
= (test
, result
, run
, time
.time(), duration
, build
, commit
)
58 conn
.execute(sql
, params
)
61 print "sqlite: " + str(e
)
62 print "sql: %r" % (params
, )
64 class DataCollector(object):
65 def __init__(self
, logdir
, testname
, tracing
, dmesg
):
67 self
._testname
= testname
68 self
._tracing
= tracing
72 output
= os
.path
.join(self
._logdir
, '%s.dat' % (self
._testname
, ))
73 self
._trace
_cmd
= subprocess
.Popen(['sudo', 'trace-cmd', 'record', '-o', output
, '-e', 'mac80211', '-e', 'cfg80211', 'sh', '-c', 'echo STARTED ; read l'],
74 stdin
=subprocess
.PIPE
,
75 stdout
=subprocess
.PIPE
,
76 stderr
=open('/dev/null', 'w'),
78 l
= self
._trace
_cmd
.stdout
.read(7)
79 while not 'STARTED' in l
:
80 l
+= self
._trace
_cmd
.stdout
.read(1)
81 def __exit__(self
, type, value
, traceback
):
83 self
._trace
_cmd
.stdin
.write('DONE\n')
84 self
._trace
_cmd
.wait()
86 output
= os
.path
.join(self
._logdir
, '%s.dmesg' % (self
._testname
, ))
87 subprocess
.call(['sudo', 'dmesg', '-c'], stdout
=open(output
, 'w'))
89 def rename_log(logdir
, basename
, testname
, dev
):
92 srcname
= os
.path
.join(logdir
, basename
)
93 dstname
= os
.path
.join(logdir
, testname
+ '.' + basename
)
95 while os
.path
.exists(dstname
):
96 dstname
= os
.path
.join(logdir
,
97 testname
+ '.' + basename
+ '-' + str(num
))
99 os
.rename(srcname
, dstname
)
102 subprocess
.call(['sudo', 'chown', '-f', getpass
.getuser(), srcname
])
104 logger
.info("Failed to rename log files")
110 for t
in os
.listdir("."):
111 m
= re
.match(r
'(test_.*)\.py$', t
)
113 logger
.debug("Import test cases from " + t
)
114 mod
= __import__(m
.group(1))
115 test_modules
.append(mod
.__name
__.replace('test_', '', 1))
117 if s
.startswith("test_"):
118 func
= mod
.__dict
__.get(s
)
120 test_names
= list(set([t
.__name
__.replace('test_', '', 1) for t
in tests
]))
124 parser
= argparse
.ArgumentParser(description
='hwsim test runner')
125 parser
.add_argument('--logdir', metavar
='<directory>',
126 help='log output directory for all other options, ' +
127 'must be given if other log options are used')
128 group
= parser
.add_mutually_exclusive_group()
129 group
.add_argument('-d', const
=logging
.DEBUG
, action
='store_const',
130 dest
='loglevel', default
=logging
.INFO
,
131 help="verbose debug output")
132 group
.add_argument('-q', const
=logging
.WARNING
, action
='store_const',
133 dest
='loglevel', help="be quiet")
135 parser
.add_argument('-S', metavar
='<sqlite3 db>', dest
='database',
136 help='database to write results to')
137 parser
.add_argument('--prefill-tests', action
='store_true', dest
='prefill',
138 help='prefill test database with NOTRUN before all tests')
139 parser
.add_argument('--commit', metavar
='<commit id>',
140 help='commit ID, only for database')
141 parser
.add_argument('-b', metavar
='<build>', dest
='build', help='build ID')
142 parser
.add_argument('-L', action
='store_true', dest
='update_tests_db',
143 help='List tests (and update descriptions in DB)')
144 parser
.add_argument('-T', action
='store_true', dest
='tracing',
145 help='collect tracing per test case (in log directory)')
146 parser
.add_argument('-D', action
='store_true', dest
='dmesg',
147 help='collect dmesg per test case (in log directory)')
148 parser
.add_argument('--shuffle-tests', action
='store_true',
149 dest
='shuffle_tests',
150 help='Shuffle test cases to randomize order')
151 parser
.add_argument('--no-reset', action
='store_true', dest
='no_reset',
152 help='Do not reset devices at the end of the test')
153 parser
.add_argument('-f', dest
='testmodules', metavar
='<test module>',
154 help='execute only tests from these test modules',
155 type=str, choices
=[[]] + test_modules
, nargs
='+')
156 parser
.add_argument('tests', metavar
='<test>', nargs
='*', type=str,
157 help='tests to run (only valid without -f)',
158 choices
=[[]] + test_names
)
160 args
= parser
.parse_args()
162 if args
.tests
and args
.testmodules
:
163 print 'Invalid arguments - both test module and tests given'
167 if os
.path
.exists('logs/current'):
168 args
.logdir
= 'logs/current'
172 # Write debug level log to a file and configurable verbosity to stdout
173 logger
.setLevel(logging
.DEBUG
)
175 stdout_handler
= logging
.StreamHandler()
176 stdout_handler
.setLevel(args
.loglevel
)
177 logger
.addHandler(stdout_handler
)
179 file_name
= os
.path
.join(args
.logdir
, 'run-tests.log')
180 log_handler
= logging
.FileHandler(file_name
)
181 log_handler
.setLevel(logging
.DEBUG
)
182 fmt
= "%(asctime)s %(levelname)s %(message)s"
183 log_formatter
= logging
.Formatter(fmt
)
184 log_handler
.setFormatter(log_formatter
)
185 logger
.addHandler(log_handler
)
189 conn
= sqlite3
.connect(args
.database
)
190 conn
.execute('CREATE TABLE IF NOT EXISTS results (test,result,run,time,duration,build,commitid)')
191 conn
.execute('CREATE TABLE IF NOT EXISTS tests (test,description)')
196 run
= int(time
.time())
198 if args
.update_tests_db
:
200 name
= t
.__name
__.replace('test_', '', 1)
201 print name
+ " - " + t
.__doc
__
203 sql
= 'INSERT OR REPLACE INTO tests(test,description) VALUES (?, ?)'
204 params
= (name
, t
.__doc
__)
206 conn
.execute(sql
, params
)
208 print "sqlite: " + str(e
)
209 print "sql: %r" % (params
,)
216 dev0
= WpaSupplicant('wlan0', '/tmp/wpas-wlan0')
217 dev1
= WpaSupplicant('wlan1', '/tmp/wpas-wlan1')
218 dev2
= WpaSupplicant('wlan2', '/tmp/wpas-wlan2')
219 dev
= [ dev0
, dev1
, dev2
]
221 apdev
.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"})
222 apdev
.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"})
226 logger
.info(d
.ifname
+ ": No response from wpa_supplicant")
228 logger
.info("DEV: " + d
.ifname
+ ": " + d
.p2p_dev_addr())
230 logger
.info("APDEV: " + ap
['ifname'])
236 # make sure nothing is left over from previous runs
237 # (if there were any other manual runs or we crashed)
238 if not reset_devs(dev
, apdev
):
245 subprocess
.call(['sudo', 'dmesg', '-c'], stdout
=open('/dev/null', 'w'))
249 name
= t
.__name
__.replace('test_', '', 1)
251 if not name
in args
.tests
:
254 if not t
.__module
__.replace('test_', '', 1) in args
.testmodules
:
256 tests_to_run
.append(t
)
258 if conn
and args
.prefill
:
259 for t
in tests_to_run
:
260 name
= t
.__name
__.replace('test_', '', 1)
261 report(conn
, False, args
.build
, args
.commit
, run
, name
, 'NOTRUN', 0)
263 if args
.shuffle_tests
:
264 from random
import shuffle
265 shuffle(tests_to_run
)
267 for t
in tests_to_run
:
268 name
= t
.__name
__.replace('test_', '', 1)
270 log_handler
.stream
.close()
271 logger
.removeHandler(log_handler
)
272 file_name
= os
.path
.join(args
.logdir
, name
+ '.log')
273 log_handler
= logging
.FileHandler(file_name
)
274 log_handler
.setLevel(logging
.DEBUG
)
275 log_handler
.setFormatter(log_formatter
)
276 logger
.addHandler(log_handler
)
279 with
DataCollector(args
.logdir
, name
, args
.tracing
, args
.dmesg
):
280 logger
.info("START " + name
)
281 if args
.loglevel
== logging
.WARNING
:
282 print "START " + name
285 logger
.info("Test: " + t
.__doc
__)
286 start
= datetime
.now()
289 d
.request("NOTE TEST-START " + name
)
291 logger
.info("Failed to issue TEST-START before " + name
+ " for " + d
.ifname
)
293 print "FAIL " + name
+ " - could not start test"
299 if t
.func_code
.co_argcount
> 1:
312 d
.request("NOTE TEST-STOP " + name
)
314 logger
.info("Failed to issue TEST-STOP after {} for {}".format(name
, d
.ifname
))
318 print "Leaving devices in current state"
320 reset_ok
= reset_devs(dev
, apdev
)
322 for i
in range(0, 3):
323 rename_log(args
.logdir
, 'log' + str(i
), name
, dev
[i
])
326 hapd
= HostapdGlobal()
328 print "Failed to connect to hostapd interface"
333 rename_log(args
.logdir
, 'hostapd', name
, hapd
)
338 if result
== 'PASS' and args
.dmesg
:
339 if not check_kernel(os
.path
.join(args
.logdir
, name
+ '.dmesg')):
344 elif result
== 'SKIP':
349 report(conn
, args
.prefill
, args
.build
, args
.commit
, run
, name
, result
, diff
.total_seconds())
350 result
= "{} {} {} {}".format(result
, name
, diff
.total_seconds(), end
)
352 if args
.loglevel
== logging
.WARNING
:
357 print "Terminating early due to device reset failure"
361 log_handler
.stream
.close()
362 logger
.removeHandler(log_handler
)
363 file_name
= os
.path
.join(args
.logdir
, 'run-tests.log')
364 log_handler
= logging
.FileHandler(file_name
)
365 log_handler
.setLevel(logging
.DEBUG
)
366 log_handler
.setFormatter(log_formatter
)
367 logger
.addHandler(log_handler
)
373 logger
.info("passed {} test case(s)".format(len(passed
)))
374 logger
.info("skipped {} test case(s)".format(len(skipped
)))
375 logger
.info("failed tests: " + str(failed
))
376 if args
.loglevel
== logging
.WARNING
:
377 print "failed tests: " + str(failed
)
379 logger
.info("passed all {} test case(s)".format(len(passed
)))
381 logger
.info("skipped {} test case(s)".format(len(skipped
)))
382 if args
.loglevel
== logging
.WARNING
:
383 print "passed all {} test case(s)".format(len(passed
))
385 print "skipped {} test case(s)".format(len(skipped
))
387 if __name__
== "__main__":