]>
Commit | Line | Data |
---|---|---|
0af7b53a RS |
1 | #!/usr/bin/python |
2 | # | |
3 | # Copyright (C) 2014 Free Software Foundation, Inc. | |
4 | # | |
5 | # This script is free software; you can redistribute it and/or modify | |
6 | # it under the terms of the GNU General Public License as published by | |
7 | # the Free Software Foundation; either version 3, or (at your option) | |
8 | # any later version. | |
9 | ||
10 | import sys | |
11 | import getopt | |
12 | import re | |
13 | from datetime import datetime | |
ae242e3f | 14 | from operator import attrgetter |
0af7b53a RS |
15 | |
16 | # True if unrecognised lines should cause a fatal error. Might want to turn | |
17 | # this on by default later. | |
18 | strict = False | |
19 | ||
20 | # True if the order of .log segments should match the .sum file, false if | |
21 | # they should keep the original order. | |
22 | sort_logs = True | |
23 | ||
24 | class Named: | |
25 | def __init__ (self, name): | |
26 | self.name = name | |
27 | ||
0af7b53a RS |
28 | class ToolRun (Named): |
29 | def __init__ (self, name): | |
30 | Named.__init__ (self, name) | |
31 | # The variations run for this tool, mapped by --target_board name. | |
32 | self.variations = dict() | |
33 | ||
34 | # Return the VariationRun for variation NAME. | |
35 | def get_variation (self, name): | |
36 | if name not in self.variations: | |
37 | self.variations[name] = VariationRun (name) | |
38 | return self.variations[name] | |
39 | ||
40 | class VariationRun (Named): | |
41 | def __init__ (self, name): | |
42 | Named.__init__ (self, name) | |
43 | # A segment of text before the harness runs start, describing which | |
44 | # baseboard files were loaded for the target. | |
45 | self.header = None | |
46 | # The harnesses run for this variation, mapped by filename. | |
47 | self.harnesses = dict() | |
48 | # A list giving the number of times each type of result has | |
49 | # been seen. | |
50 | self.counts = [] | |
51 | ||
52 | # Return the HarnessRun for harness NAME. | |
53 | def get_harness (self, name): | |
54 | if name not in self.harnesses: | |
55 | self.harnesses[name] = HarnessRun (name) | |
56 | return self.harnesses[name] | |
57 | ||
58 | class HarnessRun (Named): | |
59 | def __init__ (self, name): | |
60 | Named.__init__ (self, name) | |
61 | # Segments of text that make up the harness run, mapped by a test-based | |
62 | # key that can be used to order them. | |
63 | self.segments = dict() | |
64 | # Segments of text that make up the harness run but which have | |
65 | # no recognized test results. These are typically harnesses that | |
66 | # are completely skipped for the target. | |
67 | self.empty = [] | |
68 | # A list of results. Each entry is a pair in which the first element | |
69 | # is a unique sorting key and in which the second is the full | |
70 | # PASS/FAIL line. | |
71 | self.results = [] | |
72 | ||
73 | # Add a segment of text to the harness run. If the segment includes | |
74 | # test results, KEY is an example of one of them, and can be used to | |
75 | # combine the individual segments in order. If the segment has no | |
76 | # test results (e.g. because the harness doesn't do anything for the | |
77 | # current configuration) then KEY is None instead. In that case | |
78 | # just collect the segments in the order that we see them. | |
79 | def add_segment (self, key, segment): | |
80 | if key: | |
81 | assert key not in self.segments | |
82 | self.segments[key] = segment | |
83 | else: | |
84 | self.empty.append (segment) | |
85 | ||
86 | class Segment: | |
87 | def __init__ (self, filename, start): | |
88 | self.filename = filename | |
89 | self.start = start | |
90 | self.lines = 0 | |
91 | ||
92 | class Prog: | |
93 | def __init__ (self): | |
94 | # The variations specified on the command line. | |
95 | self.variations = [] | |
96 | # The variations seen in the input files. | |
97 | self.known_variations = set() | |
98 | # The tools specified on the command line. | |
99 | self.tools = [] | |
100 | # Whether to create .sum rather than .log output. | |
101 | self.do_sum = True | |
102 | # Regexps used while parsing. | |
103 | self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$') | |
104 | self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') | |
105 | self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' | |
106 | r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' | |
107 | r'|KFAIL):\s*(\S+)') | |
108 | self.completed_re = re.compile (r'.* completed at (.*)') | |
109 | # Pieces of text to write at the head of the output. | |
110 | # start_line is a pair in which the first element is a datetime | |
111 | # and in which the second is the associated 'Test Run By' line. | |
112 | self.start_line = None | |
113 | self.native_line = '' | |
114 | self.target_line = '' | |
115 | self.host_line = '' | |
116 | self.acats_premable = '' | |
117 | # Pieces of text to write at the end of the output. | |
118 | # end_line is like start_line but for the 'runtest completed' line. | |
119 | self.acats_failures = [] | |
120 | self.version_output = '' | |
121 | self.end_line = None | |
122 | # Known summary types. | |
123 | self.count_names = [ | |
124 | '# of expected passes\t\t', | |
125 | '# of unexpected failures\t', | |
126 | '# of unexpected successes\t', | |
127 | '# of expected failures\t\t', | |
128 | '# of unknown successes\t\t', | |
129 | '# of known failures\t\t', | |
130 | '# of untested testcases\t\t', | |
131 | '# of unresolved testcases\t', | |
132 | '# of unsupported tests\t\t' | |
133 | ] | |
134 | self.runs = dict() | |
135 | ||
136 | def usage (self): | |
137 | name = sys.argv[0] | |
138 | sys.stderr.write ('Usage: ' + name | |
139 | + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... | |
140 | ||
141 | tool The tool (e.g. g++, libffi) for which to create a | |
142 | new test summary file. If not specified then output | |
143 | is created for all tools. | |
144 | variant-list One or more test variant names. If the list is | |
145 | not specified then one is constructed from all | |
146 | variants in the files for <tool>. | |
147 | sum-file A test summary file with the format of those | |
148 | created by runtest from DejaGnu. | |
149 | If -L is used, merge *.log files instead of *.sum. In this | |
150 | mode the exact order of lines may not be preserved, just different | |
151 | Running *.exp chunks should be in correct order. | |
152 | ''') | |
153 | sys.exit (1) | |
154 | ||
155 | def fatal (self, what, string): | |
156 | if not what: | |
157 | what = sys.argv[0] | |
158 | sys.stderr.write (what + ': ' + string + '\n') | |
159 | sys.exit (1) | |
160 | ||
161 | # Parse the command-line arguments. | |
162 | def parse_cmdline (self): | |
163 | try: | |
164 | (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') | |
165 | if len (self.files) == 0: | |
166 | self.usage() | |
167 | for (option, value) in options: | |
168 | if option == '-l': | |
169 | self.variations.append (value) | |
170 | elif option == '-t': | |
171 | self.tools.append (value) | |
172 | else: | |
173 | self.do_sum = False | |
174 | except getopt.GetoptError as e: | |
175 | self.fatal (None, e.msg) | |
176 | ||
177 | # Try to parse time string TIME, returning an arbitrary time on failure. | |
178 | # Getting this right is just a nice-to-have so failures should be silent. | |
179 | def parse_time (self, time): | |
180 | try: | |
181 | return datetime.strptime (time, '%c') | |
182 | except ValueError: | |
183 | return datetime.now() | |
184 | ||
185 | # Parse an integer and abort on failure. | |
186 | def parse_int (self, filename, value): | |
187 | try: | |
188 | return int (value) | |
189 | except ValueError: | |
190 | self.fatal (filename, 'expected an integer, got: ' + value) | |
191 | ||
192 | # Return a list that represents no test results. | |
193 | def zero_counts (self): | |
194 | return [0 for x in self.count_names] | |
195 | ||
196 | # Return the ToolRun for tool NAME. | |
197 | def get_tool (self, name): | |
198 | if name not in self.runs: | |
199 | self.runs[name] = ToolRun (name) | |
200 | return self.runs[name] | |
201 | ||
202 | # Add the result counts in list FROMC to TOC. | |
203 | def accumulate_counts (self, toc, fromc): | |
204 | for i in range (len (self.count_names)): | |
205 | toc[i] += fromc[i] | |
206 | ||
207 | # Parse the list of variations after 'Schedule of variations:'. | |
208 | # Return the number seen. | |
209 | def parse_variations (self, filename, file): | |
210 | num_variations = 0 | |
211 | while True: | |
212 | line = file.readline() | |
213 | if line == '': | |
214 | self.fatal (filename, 'could not parse variation list') | |
215 | if line == '\n': | |
216 | break | |
217 | self.known_variations.add (line.strip()) | |
218 | num_variations += 1 | |
219 | return num_variations | |
220 | ||
221 | # Parse from the first line after 'Running target ...' to the end | |
222 | # of the run's summary. | |
223 | def parse_run (self, filename, file, tool, variation, num_variations): | |
224 | header = None | |
225 | harness = None | |
226 | segment = None | |
227 | final_using = 0 | |
228 | ||
229 | # If this is the first run for this variation, add any text before | |
230 | # the first harness to the header. | |
231 | if not variation.header: | |
232 | segment = Segment (filename, file.tell()) | |
233 | variation.header = segment | |
234 | ||
235 | # Parse up until the first line of the summary. | |
236 | if num_variations == 1: | |
237 | end = '\t\t=== ' + tool.name + ' Summary ===\n' | |
238 | else: | |
239 | end = ('\t\t=== ' + tool.name + ' Summary for ' | |
240 | + variation.name + ' ===\n') | |
241 | while True: | |
242 | line = file.readline() | |
243 | if line == '': | |
244 | self.fatal (filename, 'no recognised summary line') | |
245 | if line == end: | |
246 | break | |
247 | ||
248 | # Look for the start of a new harness. | |
249 | if line.startswith ('Running ') and line.endswith (' ...\n'): | |
250 | # Close off the current harness segment, if any. | |
251 | if harness: | |
252 | segment.lines -= final_using | |
253 | harness.add_segment (first_key, segment) | |
254 | name = line[len ('Running '):-len(' ...\n')] | |
255 | harness = variation.get_harness (name) | |
256 | segment = Segment (filename, file.tell()) | |
257 | first_key = None | |
258 | final_using = 0 | |
259 | continue | |
260 | ||
261 | # Record test results. Associate the first test result with | |
262 | # the harness segment, so that if a run for a particular harness | |
263 | # has been split up, we can reassemble the individual segments | |
264 | # in a sensible order. | |
768b94bc RS |
265 | # |
266 | # dejagnu sometimes issues warnings about the testing environment | |
267 | # before running any tests. Treat them as part of the header | |
268 | # rather than as a test result. | |
0af7b53a | 269 | match = self.result_re.match (line) |
768b94bc | 270 | if match and (harness or not line.startswith ('WARNING:')): |
0af7b53a RS |
271 | if not harness: |
272 | self.fatal (filename, 'saw test result before harness name') | |
273 | name = match.group (2) | |
274 | # Ugly hack to get the right order for gfortran. | |
275 | if name.startswith ('gfortran.dg/g77/'): | |
276 | name = 'h' + name | |
277 | key = (name, len (harness.results)) | |
278 | harness.results.append ((key, line)) | |
279 | if not first_key and sort_logs: | |
280 | first_key = key | |
281 | ||
282 | # 'Using ...' lines are only interesting in a header. Splitting | |
283 | # the test up into parallel runs leads to more 'Using ...' lines | |
284 | # than there would be in a single log. | |
285 | if line.startswith ('Using '): | |
286 | final_using += 1 | |
287 | else: | |
288 | final_using = 0 | |
289 | ||
290 | # Add other text to the current segment, if any. | |
291 | if segment: | |
292 | segment.lines += 1 | |
293 | ||
294 | # Close off the final harness segment, if any. | |
295 | if harness: | |
296 | segment.lines -= final_using | |
297 | harness.add_segment (first_key, segment) | |
298 | ||
299 | # Parse the rest of the summary (the '# of ' lines). | |
300 | if len (variation.counts) == 0: | |
301 | variation.counts = self.zero_counts() | |
302 | while True: | |
303 | before = file.tell() | |
304 | line = file.readline() | |
305 | if line == '': | |
306 | break | |
307 | if line == '\n': | |
308 | continue | |
309 | if not line.startswith ('# '): | |
310 | file.seek (before) | |
311 | break | |
312 | found = False | |
313 | for i in range (len (self.count_names)): | |
314 | if line.startswith (self.count_names[i]): | |
315 | count = line[len (self.count_names[i]):-1].strip() | |
316 | variation.counts[i] += self.parse_int (filename, count) | |
317 | found = True | |
318 | break | |
319 | if not found: | |
320 | self.fatal (filename, 'unknown test result: ' + line[:-1]) | |
321 | ||
322 | # Parse an acats run, which uses a different format from dejagnu. | |
323 | # We have just skipped over '=== acats configuration ==='. | |
324 | def parse_acats_run (self, filename, file): | |
325 | # Parse the preamble, which describes the configuration and logs | |
326 | # the creation of support files. | |
327 | record = (self.acats_premable == '') | |
328 | if record: | |
329 | self.acats_premable = '\t\t=== acats configuration ===\n' | |
330 | while True: | |
331 | line = file.readline() | |
332 | if line == '': | |
333 | self.fatal (filename, 'could not parse acats preamble') | |
334 | if line == '\t\t=== acats tests ===\n': | |
335 | break | |
336 | if record: | |
337 | self.acats_premable += line | |
338 | ||
339 | # Parse the test results themselves, using a dummy variation name. | |
340 | tool = self.get_tool ('acats') | |
341 | variation = tool.get_variation ('none') | |
342 | self.parse_run (filename, file, tool, variation, 1) | |
343 | ||
344 | # Parse the failure list. | |
345 | while True: | |
346 | before = file.tell() | |
347 | line = file.readline() | |
348 | if line.startswith ('*** FAILURES: '): | |
349 | self.acats_failures.append (line[len ('*** FAILURES: '):-1]) | |
350 | continue | |
351 | file.seek (before) | |
352 | break | |
353 | ||
354 | # Parse the final summary at the end of a log in order to capture | |
355 | # the version output that follows it. | |
356 | def parse_final_summary (self, filename, file): | |
357 | record = (self.version_output == '') | |
358 | while True: | |
359 | line = file.readline() | |
360 | if line == '': | |
361 | break | |
362 | if line.startswith ('# of '): | |
363 | continue | |
364 | if record: | |
365 | self.version_output += line | |
366 | if line == '\n': | |
367 | break | |
368 | ||
369 | # Parse a .log or .sum file. | |
370 | def parse_file (self, filename, file): | |
371 | tool = None | |
372 | target = None | |
373 | num_variations = 1 | |
374 | while True: | |
375 | line = file.readline() | |
376 | if line == '': | |
377 | return | |
378 | ||
379 | # Parse the list of variations, which comes before the test | |
380 | # runs themselves. | |
381 | if line.startswith ('Schedule of variations:'): | |
382 | num_variations = self.parse_variations (filename, file) | |
383 | continue | |
384 | ||
385 | # Parse a testsuite run for one tool/variation combination. | |
386 | if line.startswith ('Running target '): | |
387 | name = line[len ('Running target '):-1] | |
388 | if not tool: | |
389 | self.fatal (filename, 'could not parse tool name') | |
390 | if name not in self.known_variations: | |
391 | self.fatal (filename, 'unknown target: ' + name) | |
392 | self.parse_run (filename, file, tool, | |
393 | tool.get_variation (name), | |
394 | num_variations) | |
395 | # If there is only one variation then there is no separate | |
396 | # summary for it. Record any following version output. | |
397 | if num_variations == 1: | |
398 | self.parse_final_summary (filename, file) | |
399 | continue | |
400 | ||
401 | # Parse the start line. In the case where several files are being | |
402 | # parsed, pick the one with the earliest time. | |
403 | match = self.test_run_re.match (line) | |
404 | if match: | |
405 | time = self.parse_time (match.group (2)) | |
406 | if not self.start_line or self.start_line[0] > time: | |
407 | self.start_line = (time, line) | |
408 | continue | |
409 | ||
410 | # Parse the form used for native testing. | |
411 | if line.startswith ('Native configuration is '): | |
412 | self.native_line = line | |
413 | continue | |
414 | ||
415 | # Parse the target triplet. | |
416 | if line.startswith ('Target is '): | |
417 | self.target_line = line | |
418 | continue | |
419 | ||
420 | # Parse the host triplet. | |
421 | if line.startswith ('Host is '): | |
422 | self.host_line = line | |
423 | continue | |
424 | ||
425 | # Parse the acats premable. | |
426 | if line == '\t\t=== acats configuration ===\n': | |
427 | self.parse_acats_run (filename, file) | |
428 | continue | |
429 | ||
430 | # Parse the tool name. | |
431 | match = self.tool_re.match (line) | |
432 | if match: | |
433 | tool = self.get_tool (match.group (1)) | |
434 | continue | |
435 | ||
436 | # Skip over the final summary (which we instead create from | |
437 | # individual runs) and parse the version output. | |
438 | if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': | |
439 | if file.readline() != '\n': | |
440 | self.fatal (filename, 'expected blank line after summary') | |
441 | self.parse_final_summary (filename, file) | |
442 | continue | |
443 | ||
444 | # Parse the completion line. In the case where several files | |
445 | # are being parsed, pick the one with the latest time. | |
446 | match = self.completed_re.match (line) | |
447 | if match: | |
448 | time = self.parse_time (match.group (1)) | |
449 | if not self.end_line or self.end_line[0] < time: | |
450 | self.end_line = (time, line) | |
451 | continue | |
452 | ||
453 | # Sanity check to make sure that important text doesn't get | |
454 | # dropped accidentally. | |
455 | if strict and line.strip() != '': | |
456 | self.fatal (filename, 'unrecognised line: ' + line[:-1]) | |
457 | ||
458 | # Output a segment of text. | |
459 | def output_segment (self, segment): | |
460 | with open (segment.filename, 'r') as file: | |
461 | file.seek (segment.start) | |
462 | for i in range (segment.lines): | |
463 | sys.stdout.write (file.readline()) | |
464 | ||
465 | # Output a summary giving the number of times each type of result has | |
466 | # been seen. | |
467 | def output_summary (self, tool, counts): | |
468 | for i in range (len (self.count_names)): | |
469 | name = self.count_names[i] | |
470 | # dejagnu only prints result types that were seen at least once, | |
471 | # but acats always prints a number of unexpected failures. | |
472 | if (counts[i] > 0 | |
473 | or (tool.name == 'acats' | |
474 | and name.startswith ('# of unexpected failures'))): | |
475 | sys.stdout.write ('%s%d\n' % (name, counts[i])) | |
476 | ||
477 | # Output unified .log or .sum information for a particular variation, | |
478 | # with a summary at the end. | |
479 | def output_variation (self, tool, variation): | |
480 | self.output_segment (variation.header) | |
ae242e3f RS |
481 | for harness in sorted (variation.harnesses.values(), |
482 | key = attrgetter ('name')): | |
0af7b53a RS |
483 | sys.stdout.write ('Running ' + harness.name + ' ...\n') |
484 | if self.do_sum: | |
485 | # Keep the original test result order if there was only | |
486 | # one segment for this harness. This is needed for | |
487 | # unsorted.exp, which has unusual test names. Otherwise | |
488 | # sort the tests by test filename. If there are several | |
489 | # subtests for the same test filename (such as 'compilation', | |
490 | # 'test for excess errors', etc.) then keep the subtests | |
491 | # in the original order. | |
492 | if len (harness.segments) > 1: | |
493 | harness.results.sort() | |
494 | for (key, line) in harness.results: | |
495 | sys.stdout.write (line) | |
496 | else: | |
497 | # Rearrange the log segments into test order (but without | |
498 | # rearranging text within those segments). | |
499 | for key in sorted (harness.segments.keys()): | |
500 | self.output_segment (harness.segments[key]) | |
501 | for segment in harness.empty: | |
502 | self.output_segment (segment) | |
503 | if len (self.variations) > 1: | |
504 | sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' | |
505 | + variation.name + ' ===\n\n') | |
506 | self.output_summary (tool, variation.counts) | |
507 | ||
508 | # Output unified .log or .sum information for a particular tool, | |
509 | # with a summary at the end. | |
510 | def output_tool (self, tool): | |
511 | counts = self.zero_counts() | |
512 | if tool.name == 'acats': | |
513 | # acats doesn't use variations, so just output everything. | |
514 | # It also has a different approach to whitespace. | |
515 | sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') | |
516 | for variation in tool.variations.values(): | |
517 | self.output_variation (tool, variation) | |
518 | self.accumulate_counts (counts, variation.counts) | |
519 | sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') | |
520 | else: | |
521 | # Output the results in the usual dejagnu runtest format. | |
522 | sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' | |
523 | 'Schedule of variations:\n') | |
524 | for name in self.variations: | |
525 | if name in tool.variations: | |
526 | sys.stdout.write (' ' + name + '\n') | |
527 | sys.stdout.write ('\n') | |
528 | for name in self.variations: | |
529 | if name in tool.variations: | |
530 | variation = tool.variations[name] | |
531 | sys.stdout.write ('Running target ' | |
532 | + variation.name + '\n') | |
533 | self.output_variation (tool, variation) | |
534 | self.accumulate_counts (counts, variation.counts) | |
535 | sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') | |
536 | self.output_summary (tool, counts) | |
537 | ||
538 | def main (self): | |
539 | self.parse_cmdline() | |
540 | try: | |
541 | # Parse the input files. | |
542 | for filename in self.files: | |
543 | with open (filename, 'r') as file: | |
544 | self.parse_file (filename, file) | |
545 | ||
546 | # Decide what to output. | |
547 | if len (self.variations) == 0: | |
548 | self.variations = sorted (self.known_variations) | |
549 | else: | |
550 | for name in self.variations: | |
551 | if name not in self.known_variations: | |
552 | self.fatal (None, 'no results for ' + name) | |
553 | if len (self.tools) == 0: | |
554 | self.tools = sorted (self.runs.keys()) | |
555 | ||
556 | # Output the header. | |
557 | if self.start_line: | |
558 | sys.stdout.write (self.start_line[1]) | |
559 | sys.stdout.write (self.native_line) | |
560 | sys.stdout.write (self.target_line) | |
561 | sys.stdout.write (self.host_line) | |
562 | sys.stdout.write (self.acats_premable) | |
563 | ||
564 | # Output the main body. | |
565 | for name in self.tools: | |
566 | if name not in self.runs: | |
567 | self.fatal (None, 'no results for ' + name) | |
568 | self.output_tool (self.runs[name]) | |
569 | ||
570 | # Output the footer. | |
571 | if len (self.acats_failures) > 0: | |
572 | sys.stdout.write ('*** FAILURES: ' | |
573 | + ' '.join (self.acats_failures) + '\n') | |
574 | sys.stdout.write (self.version_output) | |
575 | if self.end_line: | |
576 | sys.stdout.write (self.end_line[1]) | |
577 | except IOError as e: | |
578 | self.fatal (e.filename, e.strerror) | |
579 | ||
580 | Prog().main() |