]>
Commit | Line | Data |
---|---|---|
fb6a751f SDJ |
1 | #!/usr/bin/env python |
2 | ||
3666a048 | 3 | # Copyright (C) 2016-2021 Free Software Foundation, Inc. |
fb6a751f SDJ |
4 | # |
5 | # This file is part of GDB. | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3 of the License, or | |
10 | # (at your option) any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | ||
20 | ||
21 | # This program is used to analyze the test results (i.e., *.sum files) | |
22 | # generated by GDB's testsuite, and print the testcases that are found | |
23 | # to be racy. | |
24 | # | |
25 | # Racy testcases are considered as being testcases which can | |
26 | # intermittently FAIL (or PASS) when run two or more times | |
27 | # consecutively, i.e., tests whose results are not deterministic. | |
28 | # | |
29 | # This program is invoked when the user runs "make check" and | |
30 | # specifies the RACY_ITER environment variable. | |
31 | ||
32 | import sys | |
33 | import os | |
34 | import re | |
35 | ||
36 | # The (global) dictionary that stores the associations between a *.sum | |
37 | # file and its results. The data inside it will be stored as: | |
38 | # | |
39 | # files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... }, | |
40 | # 'FAIL' : { 'test5', 'test6' ... }, | |
41 | # ... | |
42 | # }, | |
43 | # { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... }, | |
44 | # ... | |
45 | # } | |
46 | # } | |
47 | ||
13123da8 | 48 | files_and_tests = dict() |
fb6a751f SDJ |
49 | |
50 | # The relatioships between various states of the same tests that | |
51 | # should be ignored. For example, if the same test PASSes on a | |
52 | # testcase run but KFAILs on another, this test should be considered | |
53 | # racy because a known-failure is... known. | |
54 | ||
13123da8 | 55 | ignore_relations = {"PASS": "KFAIL"} |
fb6a751f SDJ |
56 | |
57 | # We are interested in lines that start with '.?(PASS|FAIL)'. In | |
58 | # other words, we don't process errors (maybe we should). | |
59 | ||
13123da8 | 60 | sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$") |
fb6a751f | 61 | |
13123da8 SM |
62 | |
63 | def parse_sum_line(line, dic): | |
fb6a751f | 64 | """Parse a single LINE from a sumfile, and store the results in the |
13123da8 | 65 | dictionary referenced by DIC.""" |
fb6a751f SDJ |
66 | global sum_matcher |
67 | ||
13123da8 SM |
68 | line = line.rstrip() |
69 | m = re.match(sum_matcher, line) | |
fb6a751f | 70 | if m: |
13123da8 SM |
71 | result = m.group(1) |
72 | test_name = m.group(3) | |
fb6a751f SDJ |
73 | # Remove tail parentheses. These are likely to be '(timeout)' |
74 | # and other extra information that will only confuse us. | |
13123da8 SM |
75 | test_name = re.sub("(\s+)?\(.*$", "", test_name) |
76 | if result not in dic.keys(): | |
77 | dic[result] = set() | |
fb6a751f SDJ |
78 | if test_name in dic[result]: |
79 | # If the line is already present in the dictionary, then | |
80 | # we include a unique identifier in the end of it, in the | |
81 | # form or '<<N>>' (where N is a number >= 2). This is | |
82 | # useful because the GDB testsuite is full of non-unique | |
83 | # test messages; however, if you process the racy summary | |
84 | # file you will also need to perform this same operation | |
85 | # in order to identify the racy test. | |
86 | i = 2 | |
87 | while True: | |
13123da8 | 88 | nname = test_name + " <<" + str(i) + ">>" |
fb6a751f SDJ |
89 | if nname not in dic[result]: |
90 | break | |
91 | i += 1 | |
92 | test_name = nname | |
13123da8 SM |
93 | dic[result].add(test_name) |
94 | ||
fb6a751f | 95 | |
13123da8 | 96 | def read_sum_files(files): |
fb6a751f | 97 | """Read the sumfiles (passed as a list in the FILES variable), and |
13123da8 SM |
98 | process each one, filling the FILES_AND_TESTS global dictionary with |
99 | information about them.""" | |
fb6a751f SDJ |
100 | global files_and_tests |
101 | ||
102 | for x in files: | |
13123da8 SM |
103 | with open(x, "r") as f: |
104 | files_and_tests[x] = dict() | |
105 | for line in f.readlines(): | |
106 | parse_sum_line(line, files_and_tests[x]) | |
fb6a751f | 107 | |
13123da8 SM |
108 | |
109 | def identify_racy_tests(): | |
fb6a751f | 110 | """Identify and print the racy tests. This function basically works |
13123da8 SM |
111 | on sets, and the idea behind it is simple. It takes all the sets that |
112 | refer to the same result (for example, all the sets that contain PASS | |
113 | tests), and compare them. If a test is present in all PASS sets, then | |
114 | it is not racy. Otherwise, it is. | |
115 | ||
116 | This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.), | |
117 | and then print a sorted list (without duplicates) of all the tests | |
118 | that were found to be racy.""" | |
fb6a751f SDJ |
119 | global files_and_tests |
120 | ||
121 | # First, construct two dictionaries that will hold one set of | |
122 | # testcases for each state (PASS, FAIL, etc.). | |
123 | # | |
124 | # Each set in NONRACY_TESTS will contain only the non-racy | |
125 | # testcases for that state. A non-racy testcase is a testcase | |
126 | # that has the same state in all test runs. | |
127 | # | |
128 | # Each set in ALL_TESTS will contain all tests, racy or not, for | |
129 | # that state. | |
13123da8 SM |
130 | nonracy_tests = dict() |
131 | all_tests = dict() | |
fb6a751f SDJ |
132 | for f in files_and_tests: |
133 | for state in files_and_tests[f]: | |
134 | try: | |
13123da8 | 135 | nonracy_tests[state] &= files_and_tests[f][state].copy() |
fb6a751f | 136 | except KeyError: |
13123da8 | 137 | nonracy_tests[state] = files_and_tests[f][state].copy() |
fb6a751f SDJ |
138 | |
139 | try: | |
13123da8 | 140 | all_tests[state] |= files_and_tests[f][state].copy() |
fb6a751f | 141 | except KeyError: |
13123da8 | 142 | all_tests[state] = files_and_tests[f][state].copy() |
fb6a751f SDJ |
143 | |
144 | # Now, we eliminate the tests that are present in states that need | |
145 | # to be ignored. For example, tests both in the PASS and KFAIL | |
146 | # states should not be considered racy. | |
13123da8 SM |
147 | ignored_tests = set() |
148 | for s1, s2 in ignore_relations.iteritems(): | |
fb6a751f | 149 | try: |
13123da8 | 150 | ignored_tests |= all_tests[s1] & all_tests[s2] |
fb6a751f SDJ |
151 | except: |
152 | continue | |
153 | ||
13123da8 | 154 | racy_tests = set() |
fb6a751f SDJ |
155 | for f in files_and_tests: |
156 | for state in files_and_tests[f]: | |
157 | racy_tests |= files_and_tests[f][state] - nonracy_tests[state] | |
158 | ||
159 | racy_tests = racy_tests - ignored_tests | |
160 | ||
161 | # Print the header. | |
162 | print "\t\t=== gdb racy tests ===\n" | |
163 | ||
164 | # Print each test. | |
13123da8 | 165 | for line in sorted(racy_tests): |
fb6a751f SDJ |
166 | print line |
167 | ||
168 | # Print the summary. | |
169 | print "\n" | |
170 | print "\t\t=== gdb Summary ===\n" | |
13123da8 SM |
171 | print "# of racy tests:\t\t%d" % len(racy_tests) |
172 | ||
fb6a751f | 173 | |
13123da8 SM |
174 | if __name__ == "__main__": |
175 | if len(sys.argv) < 3: | |
fb6a751f SDJ |
176 | # It only makes sense to invoke this program if you pass two |
177 | # or more files to be analyzed. | |
13123da8 SM |
178 | sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0]) |
179 | read_sum_files(sys.argv[1:]) | |
180 | identify_racy_tests() | |
181 | exit(0) |