]>
Commit | Line | Data |
---|---|---|
d201506c SW |
1 | # Copyright (c) 2015 Stephen Warren |
2 | # Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. | |
3 | # | |
4 | # SPDX-License-Identifier: GPL-2.0 | |
5 | ||
6 | # Implementation of pytest run-time hook functions. These are invoked by | |
7 | # pytest at certain points during operation, e.g. startup, for each executed | |
8 | # test, at shutdown etc. These hooks perform functions such as: | |
9 | # - Parsing custom command-line options. | |
10 | # - Pullilng in user-specified board configuration. | |
11 | # - Creating the U-Boot console test fixture. | |
12 | # - Creating the HTML log file. | |
13 | # - Monitoring each test's results. | |
14 | # - Implementing custom pytest markers. | |
15 | ||
16 | import atexit | |
17 | import errno | |
18 | import os | |
19 | import os.path | |
20 | import pexpect | |
21 | import pytest | |
22 | from _pytest.runner import runtestprotocol | |
23 | import ConfigParser | |
24 | import StringIO | |
25 | import sys | |
26 | ||
27 | # Globals: The HTML log file, and the connection to the U-Boot console. | |
28 | log = None | |
29 | console = None | |
30 | ||
31 | def mkdir_p(path): | |
e8debf39 | 32 | """Create a directory path. |
d201506c SW |
33 | |
34 | This includes creating any intermediate/parent directories. Any errors | |
35 | caused due to already extant directories are ignored. | |
36 | ||
37 | Args: | |
38 | path: The directory path to create. | |
39 | ||
40 | Returns: | |
41 | Nothing. | |
e8debf39 | 42 | """ |
d201506c SW |
43 | |
44 | try: | |
45 | os.makedirs(path) | |
46 | except OSError as exc: | |
47 | if exc.errno == errno.EEXIST and os.path.isdir(path): | |
48 | pass | |
49 | else: | |
50 | raise | |
51 | ||
52 | def pytest_addoption(parser): | |
e8debf39 | 53 | """pytest hook: Add custom command-line options to the cmdline parser. |
d201506c SW |
54 | |
55 | Args: | |
56 | parser: The pytest command-line parser. | |
57 | ||
58 | Returns: | |
59 | Nothing. | |
e8debf39 | 60 | """ |
d201506c SW |
61 | |
62 | parser.addoption('--build-dir', default=None, | |
63 | help='U-Boot build directory (O=)') | |
64 | parser.addoption('--result-dir', default=None, | |
65 | help='U-Boot test result/tmp directory') | |
66 | parser.addoption('--persistent-data-dir', default=None, | |
67 | help='U-Boot test persistent generated data directory') | |
68 | parser.addoption('--board-type', '--bd', '-B', default='sandbox', | |
69 | help='U-Boot board type') | |
70 | parser.addoption('--board-identity', '--id', default='na', | |
71 | help='U-Boot board identity/instance') | |
72 | parser.addoption('--build', default=False, action='store_true', | |
73 | help='Compile U-Boot before running tests') | |
74 | ||
75 | def pytest_configure(config): | |
e8debf39 | 76 | """pytest hook: Perform custom initialization at startup time. |
d201506c SW |
77 | |
78 | Args: | |
79 | config: The pytest configuration. | |
80 | ||
81 | Returns: | |
82 | Nothing. | |
e8debf39 | 83 | """ |
d201506c SW |
84 | |
85 | global log | |
86 | global console | |
87 | global ubconfig | |
88 | ||
89 | test_py_dir = os.path.dirname(os.path.abspath(__file__)) | |
90 | source_dir = os.path.dirname(os.path.dirname(test_py_dir)) | |
91 | ||
92 | board_type = config.getoption('board_type') | |
93 | board_type_filename = board_type.replace('-', '_') | |
94 | ||
95 | board_identity = config.getoption('board_identity') | |
96 | board_identity_filename = board_identity.replace('-', '_') | |
97 | ||
98 | build_dir = config.getoption('build_dir') | |
99 | if not build_dir: | |
100 | build_dir = source_dir + '/build-' + board_type | |
101 | mkdir_p(build_dir) | |
102 | ||
103 | result_dir = config.getoption('result_dir') | |
104 | if not result_dir: | |
105 | result_dir = build_dir | |
106 | mkdir_p(result_dir) | |
107 | ||
108 | persistent_data_dir = config.getoption('persistent_data_dir') | |
109 | if not persistent_data_dir: | |
110 | persistent_data_dir = build_dir + '/persistent-data' | |
111 | mkdir_p(persistent_data_dir) | |
112 | ||
113 | import multiplexed_log | |
114 | log = multiplexed_log.Logfile(result_dir + '/test-log.html') | |
115 | ||
116 | if config.getoption('build'): | |
117 | if build_dir != source_dir: | |
118 | o_opt = 'O=%s' % build_dir | |
119 | else: | |
120 | o_opt = '' | |
121 | cmds = ( | |
122 | ['make', o_opt, '-s', board_type + '_defconfig'], | |
123 | ['make', o_opt, '-s', '-j8'], | |
124 | ) | |
125 | runner = log.get_runner('make', sys.stdout) | |
126 | for cmd in cmds: | |
127 | runner.run(cmd, cwd=source_dir) | |
128 | runner.close() | |
129 | ||
130 | class ArbitraryAttributeContainer(object): | |
131 | pass | |
132 | ||
133 | ubconfig = ArbitraryAttributeContainer() | |
134 | ubconfig.brd = dict() | |
135 | ubconfig.env = dict() | |
136 | ||
137 | modules = [ | |
138 | (ubconfig.brd, 'u_boot_board_' + board_type_filename), | |
139 | (ubconfig.env, 'u_boot_boardenv_' + board_type_filename), | |
140 | (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' + | |
141 | board_identity_filename), | |
142 | ] | |
143 | for (dict_to_fill, module_name) in modules: | |
144 | try: | |
145 | module = __import__(module_name) | |
146 | except ImportError: | |
147 | continue | |
148 | dict_to_fill.update(module.__dict__) | |
149 | ||
150 | ubconfig.buildconfig = dict() | |
151 | ||
152 | for conf_file in ('.config', 'include/autoconf.mk'): | |
153 | dot_config = build_dir + '/' + conf_file | |
154 | if not os.path.exists(dot_config): | |
155 | raise Exception(conf_file + ' does not exist; ' + | |
156 | 'try passing --build option?') | |
157 | ||
158 | with open(dot_config, 'rt') as f: | |
159 | ini_str = '[root]\n' + f.read() | |
160 | ini_sio = StringIO.StringIO(ini_str) | |
161 | parser = ConfigParser.RawConfigParser() | |
162 | parser.readfp(ini_sio) | |
163 | ubconfig.buildconfig.update(parser.items('root')) | |
164 | ||
165 | ubconfig.test_py_dir = test_py_dir | |
166 | ubconfig.source_dir = source_dir | |
167 | ubconfig.build_dir = build_dir | |
168 | ubconfig.result_dir = result_dir | |
169 | ubconfig.persistent_data_dir = persistent_data_dir | |
170 | ubconfig.board_type = board_type | |
171 | ubconfig.board_identity = board_identity | |
172 | ||
173 | env_vars = ( | |
174 | 'board_type', | |
175 | 'board_identity', | |
176 | 'source_dir', | |
177 | 'test_py_dir', | |
178 | 'build_dir', | |
179 | 'result_dir', | |
180 | 'persistent_data_dir', | |
181 | ) | |
182 | for v in env_vars: | |
183 | os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v) | |
184 | ||
185 | if board_type == 'sandbox': | |
186 | import u_boot_console_sandbox | |
187 | console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig) | |
188 | else: | |
189 | import u_boot_console_exec_attach | |
190 | console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig) | |
191 | ||
192 | def pytest_generate_tests(metafunc): | |
e8debf39 | 193 | """pytest hook: parameterize test functions based on custom rules. |
d201506c SW |
194 | |
195 | If a test function takes parameter(s) (fixture names) of the form brd__xxx | |
196 | or env__xxx, the brd and env configuration dictionaries are consulted to | |
197 | find the list of values to use for those parameters, and the test is | |
198 | parametrized so that it runs once for each combination of values. | |
199 | ||
200 | Args: | |
201 | metafunc: The pytest test function. | |
202 | ||
203 | Returns: | |
204 | Nothing. | |
e8debf39 | 205 | """ |
d201506c SW |
206 | |
207 | subconfigs = { | |
208 | 'brd': console.config.brd, | |
209 | 'env': console.config.env, | |
210 | } | |
211 | for fn in metafunc.fixturenames: | |
212 | parts = fn.split('__') | |
213 | if len(parts) < 2: | |
214 | continue | |
215 | if parts[0] not in subconfigs: | |
216 | continue | |
217 | subconfig = subconfigs[parts[0]] | |
218 | vals = [] | |
219 | val = subconfig.get(fn, []) | |
220 | # If that exact name is a key in the data source: | |
221 | if val: | |
222 | # ... use the dict value as a single parameter value. | |
223 | vals = (val, ) | |
224 | else: | |
225 | # ... otherwise, see if there's a key that contains a list of | |
226 | # values to use instead. | |
227 | vals = subconfig.get(fn + 's', []) | |
d20e5e97 SW |
228 | def fixture_id(index, val): |
229 | try: | |
230 | return val["fixture_id"] | |
231 | except: | |
232 | return fn + str(index) | |
233 | ids = [fixture_id(index, val) for (index, val) in enumerate(vals)] | |
234 | metafunc.parametrize(fn, vals, ids=ids) | |
d201506c | 235 | |
636f38d8 | 236 | @pytest.fixture(scope='function') |
d201506c | 237 | def u_boot_console(request): |
e8debf39 | 238 | """Generate the value of a test's u_boot_console fixture. |
d201506c SW |
239 | |
240 | Args: | |
241 | request: The pytest request. | |
242 | ||
243 | Returns: | |
244 | The fixture value. | |
e8debf39 | 245 | """ |
d201506c | 246 | |
636f38d8 | 247 | console.ensure_spawned() |
d201506c SW |
248 | return console |
249 | ||
250 | tests_not_run = set() | |
251 | tests_failed = set() | |
78b39cc3 SW |
252 | tests_xpassed = set() |
253 | tests_xfailed = set() | |
d201506c SW |
254 | tests_skipped = set() |
255 | tests_passed = set() | |
256 | ||
257 | def pytest_itemcollected(item): | |
e8debf39 | 258 | """pytest hook: Called once for each test found during collection. |
d201506c SW |
259 | |
260 | This enables our custom result analysis code to see the list of all tests | |
261 | that should eventually be run. | |
262 | ||
263 | Args: | |
264 | item: The item that was collected. | |
265 | ||
266 | Returns: | |
267 | Nothing. | |
e8debf39 | 268 | """ |
d201506c SW |
269 | |
270 | tests_not_run.add(item.name) | |
271 | ||
272 | def cleanup(): | |
e8debf39 | 273 | """Clean up all global state. |
d201506c SW |
274 | |
275 | Executed (via atexit) once the entire test process is complete. This | |
276 | includes logging the status of all tests, and the identity of any failed | |
277 | or skipped tests. | |
278 | ||
279 | Args: | |
280 | None. | |
281 | ||
282 | Returns: | |
283 | Nothing. | |
e8debf39 | 284 | """ |
d201506c SW |
285 | |
286 | if console: | |
287 | console.close() | |
288 | if log: | |
289 | log.status_pass('%d passed' % len(tests_passed)) | |
290 | if tests_skipped: | |
291 | log.status_skipped('%d skipped' % len(tests_skipped)) | |
292 | for test in tests_skipped: | |
293 | log.status_skipped('... ' + test) | |
78b39cc3 SW |
294 | if tests_xpassed: |
295 | log.status_xpass('%d xpass' % len(tests_xpassed)) | |
296 | for test in tests_xpassed: | |
297 | log.status_xpass('... ' + test) | |
298 | if tests_xfailed: | |
299 | log.status_xfail('%d xfail' % len(tests_xfailed)) | |
300 | for test in tests_xfailed: | |
301 | log.status_xfail('... ' + test) | |
d201506c SW |
302 | if tests_failed: |
303 | log.status_fail('%d failed' % len(tests_failed)) | |
304 | for test in tests_failed: | |
305 | log.status_fail('... ' + test) | |
306 | if tests_not_run: | |
307 | log.status_fail('%d not run' % len(tests_not_run)) | |
308 | for test in tests_not_run: | |
309 | log.status_fail('... ' + test) | |
310 | log.close() | |
311 | atexit.register(cleanup) | |
312 | ||
313 | def setup_boardspec(item): | |
e8debf39 | 314 | """Process any 'boardspec' marker for a test. |
d201506c SW |
315 | |
316 | Such a marker lists the set of board types that a test does/doesn't | |
317 | support. If tests are being executed on an unsupported board, the test is | |
318 | marked to be skipped. | |
319 | ||
320 | Args: | |
321 | item: The pytest test item. | |
322 | ||
323 | Returns: | |
324 | Nothing. | |
e8debf39 | 325 | """ |
d201506c SW |
326 | |
327 | mark = item.get_marker('boardspec') | |
328 | if not mark: | |
329 | return | |
330 | required_boards = [] | |
331 | for board in mark.args: | |
332 | if board.startswith('!'): | |
333 | if ubconfig.board_type == board[1:]: | |
334 | pytest.skip('board not supported') | |
335 | return | |
336 | else: | |
337 | required_boards.append(board) | |
338 | if required_boards and ubconfig.board_type not in required_boards: | |
339 | pytest.skip('board not supported') | |
340 | ||
341 | def setup_buildconfigspec(item): | |
e8debf39 | 342 | """Process any 'buildconfigspec' marker for a test. |
d201506c SW |
343 | |
344 | Such a marker lists some U-Boot configuration feature that the test | |
345 | requires. If tests are being executed on an U-Boot build that doesn't | |
346 | have the required feature, the test is marked to be skipped. | |
347 | ||
348 | Args: | |
349 | item: The pytest test item. | |
350 | ||
351 | Returns: | |
352 | Nothing. | |
e8debf39 | 353 | """ |
d201506c SW |
354 | |
355 | mark = item.get_marker('buildconfigspec') | |
356 | if not mark: | |
357 | return | |
358 | for option in mark.args: | |
359 | if not ubconfig.buildconfig.get('config_' + option.lower(), None): | |
360 | pytest.skip('.config feature not enabled') | |
361 | ||
362 | def pytest_runtest_setup(item): | |
e8debf39 | 363 | """pytest hook: Configure (set up) a test item. |
d201506c SW |
364 | |
365 | Called once for each test to perform any custom configuration. This hook | |
366 | is used to skip the test if certain conditions apply. | |
367 | ||
368 | Args: | |
369 | item: The pytest test item. | |
370 | ||
371 | Returns: | |
372 | Nothing. | |
e8debf39 | 373 | """ |
d201506c SW |
374 | |
375 | log.start_section(item.name) | |
376 | setup_boardspec(item) | |
377 | setup_buildconfigspec(item) | |
378 | ||
379 | def pytest_runtest_protocol(item, nextitem): | |
e8debf39 | 380 | """pytest hook: Called to execute a test. |
d201506c SW |
381 | |
382 | This hook wraps the standard pytest runtestprotocol() function in order | |
383 | to acquire visibility into, and record, each test function's result. | |
384 | ||
385 | Args: | |
386 | item: The pytest test item to execute. | |
387 | nextitem: The pytest test item that will be executed after this one. | |
388 | ||
389 | Returns: | |
390 | A list of pytest reports (test result data). | |
e8debf39 | 391 | """ |
d201506c SW |
392 | |
393 | reports = runtestprotocol(item, nextitem=nextitem) | |
78b39cc3 SW |
394 | |
395 | failure_cleanup = False | |
396 | test_list = tests_passed | |
397 | msg = 'OK' | |
398 | msg_log = log.status_pass | |
d201506c SW |
399 | for report in reports: |
400 | if report.outcome == 'failed': | |
78b39cc3 SW |
401 | if hasattr(report, 'wasxfail'): |
402 | test_list = tests_xpassed | |
403 | msg = 'XPASSED' | |
404 | msg_log = log.status_xpass | |
405 | else: | |
406 | failure_cleanup = True | |
407 | test_list = tests_failed | |
408 | msg = 'FAILED:\n' + str(report.longrepr) | |
409 | msg_log = log.status_fail | |
d201506c SW |
410 | break |
411 | if report.outcome == 'skipped': | |
78b39cc3 SW |
412 | if hasattr(report, 'wasxfail'): |
413 | failure_cleanup = True | |
414 | test_list = tests_xfailed | |
415 | msg = 'XFAILED:\n' + str(report.longrepr) | |
416 | msg_log = log.status_xfail | |
417 | break | |
418 | test_list = tests_skipped | |
419 | msg = 'SKIPPED:\n' + str(report.longrepr) | |
420 | msg_log = log.status_skipped | |
421 | ||
422 | if failure_cleanup: | |
c10eb9d3 | 423 | console.drain_console() |
78b39cc3 SW |
424 | |
425 | test_list.add(item.name) | |
d201506c SW |
426 | tests_not_run.remove(item.name) |
427 | ||
428 | try: | |
78b39cc3 | 429 | msg_log(msg) |
d201506c SW |
430 | except: |
431 | # If something went wrong with logging, it's better to let the test | |
432 | # process continue, which may report other exceptions that triggered | |
433 | # the logging issue (e.g. console.log wasn't created). Hence, just | |
434 | # squash the exception. If the test setup failed due to e.g. syntax | |
435 | # error somewhere else, this won't be seen. However, once that issue | |
436 | # is fixed, if this exception still exists, it will then be logged as | |
437 | # part of the test's stdout. | |
438 | import traceback | |
439 | print 'Exception occurred while logging runtest status:' | |
440 | traceback.print_exc() | |
441 | # FIXME: Can we force a test failure here? | |
442 | ||
443 | log.end_section(item.name) | |
444 | ||
78b39cc3 | 445 | if failure_cleanup: |
d201506c SW |
446 | console.cleanup_spawn() |
447 | ||
448 | return reports |