]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/env python | |
2 | ||
3 | # Copyright (C) 2016-2021 Free Software Foundation, Inc. | |
4 | # | |
5 | # This file is part of GDB. | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 3 of the License, or | |
10 | # (at your option) any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | ||
20 | ||
21 | # This program is used to analyze the test results (i.e., *.sum files) | |
22 | # generated by GDB's testsuite, and print the testcases that are found | |
23 | # to be racy. | |
24 | # | |
25 | # Racy testcases are considered as being testcases which can | |
26 | # intermittently FAIL (or PASS) when run two or more times | |
27 | # consecutively, i.e., tests whose results are not deterministic. | |
28 | # | |
29 | # This program is invoked when the user runs "make check" and | |
30 | # specifies the RACY_ITER environment variable. | |
31 | ||
32 | import sys | |
33 | import os | |
34 | import re | |
35 | ||
36 | # The (global) dictionary that stores the associations between a *.sum | |
37 | # file and its results. The data inside it will be stored as: | |
38 | # | |
39 | # files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... }, | |
40 | # 'FAIL' : { 'test5', 'test6' ... }, | |
41 | # ... | |
42 | # }, | |
43 | # { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... }, | |
44 | # ... | |
45 | # } | |
46 | # } | |
47 | ||
48 | files_and_tests = dict() | |
49 | ||
50 | # The relatioships between various states of the same tests that | |
51 | # should be ignored. For example, if the same test PASSes on a | |
52 | # testcase run but KFAILs on another, this test should be considered | |
53 | # racy because a known-failure is... known. | |
54 | ||
55 | ignore_relations = {"PASS": "KFAIL"} | |
56 | ||
57 | # We are interested in lines that start with '.?(PASS|FAIL)'. In | |
58 | # other words, we don't process errors (maybe we should). | |
59 | ||
60 | sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$") | |
61 | ||
62 | ||
63 | def parse_sum_line(line, dic): | |
64 | """Parse a single LINE from a sumfile, and store the results in the | |
65 | dictionary referenced by DIC.""" | |
66 | global sum_matcher | |
67 | ||
68 | line = line.rstrip() | |
69 | m = re.match(sum_matcher, line) | |
70 | if m: | |
71 | result = m.group(1) | |
72 | test_name = m.group(3) | |
73 | # Remove tail parentheses. These are likely to be '(timeout)' | |
74 | # and other extra information that will only confuse us. | |
75 | test_name = re.sub("(\s+)?\(.*$", "", test_name) | |
76 | if result not in dic.keys(): | |
77 | dic[result] = set() | |
78 | if test_name in dic[result]: | |
79 | # If the line is already present in the dictionary, then | |
80 | # we include a unique identifier in the end of it, in the | |
81 | # form or '<<N>>' (where N is a number >= 2). This is | |
82 | # useful because the GDB testsuite is full of non-unique | |
83 | # test messages; however, if you process the racy summary | |
84 | # file you will also need to perform this same operation | |
85 | # in order to identify the racy test. | |
86 | i = 2 | |
87 | while True: | |
88 | nname = test_name + " <<" + str(i) + ">>" | |
89 | if nname not in dic[result]: | |
90 | break | |
91 | i += 1 | |
92 | test_name = nname | |
93 | dic[result].add(test_name) | |
94 | ||
95 | ||
96 | def read_sum_files(files): | |
97 | """Read the sumfiles (passed as a list in the FILES variable), and | |
98 | process each one, filling the FILES_AND_TESTS global dictionary with | |
99 | information about them.""" | |
100 | global files_and_tests | |
101 | ||
102 | for x in files: | |
103 | with open(x, "r") as f: | |
104 | files_and_tests[x] = dict() | |
105 | for line in f.readlines(): | |
106 | parse_sum_line(line, files_and_tests[x]) | |
107 | ||
108 | ||
109 | def identify_racy_tests(): | |
110 | """Identify and print the racy tests. This function basically works | |
111 | on sets, and the idea behind it is simple. It takes all the sets that | |
112 | refer to the same result (for example, all the sets that contain PASS | |
113 | tests), and compare them. If a test is present in all PASS sets, then | |
114 | it is not racy. Otherwise, it is. | |
115 | ||
116 | This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.), | |
117 | and then print a sorted list (without duplicates) of all the tests | |
118 | that were found to be racy.""" | |
119 | global files_and_tests | |
120 | ||
121 | # First, construct two dictionaries that will hold one set of | |
122 | # testcases for each state (PASS, FAIL, etc.). | |
123 | # | |
124 | # Each set in NONRACY_TESTS will contain only the non-racy | |
125 | # testcases for that state. A non-racy testcase is a testcase | |
126 | # that has the same state in all test runs. | |
127 | # | |
128 | # Each set in ALL_TESTS will contain all tests, racy or not, for | |
129 | # that state. | |
130 | nonracy_tests = dict() | |
131 | all_tests = dict() | |
132 | for f in files_and_tests: | |
133 | for state in files_and_tests[f]: | |
134 | try: | |
135 | nonracy_tests[state] &= files_and_tests[f][state].copy() | |
136 | except KeyError: | |
137 | nonracy_tests[state] = files_and_tests[f][state].copy() | |
138 | ||
139 | try: | |
140 | all_tests[state] |= files_and_tests[f][state].copy() | |
141 | except KeyError: | |
142 | all_tests[state] = files_and_tests[f][state].copy() | |
143 | ||
144 | # Now, we eliminate the tests that are present in states that need | |
145 | # to be ignored. For example, tests both in the PASS and KFAIL | |
146 | # states should not be considered racy. | |
147 | ignored_tests = set() | |
148 | for s1, s2 in ignore_relations.iteritems(): | |
149 | try: | |
150 | ignored_tests |= all_tests[s1] & all_tests[s2] | |
151 | except: | |
152 | continue | |
153 | ||
154 | racy_tests = set() | |
155 | for f in files_and_tests: | |
156 | for state in files_and_tests[f]: | |
157 | racy_tests |= files_and_tests[f][state] - nonracy_tests[state] | |
158 | ||
159 | racy_tests = racy_tests - ignored_tests | |
160 | ||
161 | # Print the header. | |
162 | print "\t\t=== gdb racy tests ===\n" | |
163 | ||
164 | # Print each test. | |
165 | for line in sorted(racy_tests): | |
166 | print line | |
167 | ||
168 | # Print the summary. | |
169 | print "\n" | |
170 | print "\t\t=== gdb Summary ===\n" | |
171 | print "# of racy tests:\t\t%d" % len(racy_tests) | |
172 | ||
173 | ||
174 | if __name__ == "__main__": | |
175 | if len(sys.argv) < 3: | |
176 | # It only makes sense to invoke this program if you pass two | |
177 | # or more files to be analyzed. | |
178 | sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0]) | |
179 | read_sum_files(sys.argv[1:]) | |
180 | identify_racy_tests() | |
181 | exit(0) |