]>
git.ipfire.org Git - thirdparty/gcc.git/blob - contrib/testsuite-management/validate_failures.py
3 # Script to compare testsuite failures against a list of known-to-fail
6 # NOTE: This script is used in installations that are running Python 2.4.
7 # Please stick to syntax features available in 2.4 and earlier
10 # Contributed by Diego Novillo <dnovillo@google.com>
12 # Copyright (C) 2011-2013 Free Software Foundation, Inc.
14 # This file is part of GCC.
16 # GCC is free software; you can redistribute it and/or modify
17 # it under the terms of the GNU General Public License as published by
18 # the Free Software Foundation; either version 3, or (at your option)
21 # GCC is distributed in the hope that it will be useful,
22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 # GNU General Public License for more details.
26 # You should have received a copy of the GNU General Public License
27 # along with GCC; see the file COPYING. If not, write to
28 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
29 # Boston, MA 02110-1301, USA.
31 """This script provides a coarser XFAILing mechanism that requires no
32 detailed DejaGNU markings. This is useful in a variety of scenarios:
34 - Development branches with many known failures waiting to be fixed.
35 - Release branches with known failures that are not considered
36 important for the particular release criteria used in that branch.
38 The script must be executed from the toplevel build directory. When
41 1- Determine the target built: TARGET
42 2- Determine the source directory: SRCDIR
43 3- Look for a failure manifest file in
44 <SRCDIR>/<MANIFEST_SUBDIR>/<MANIFEST_NAME>.xfail
45 4- Collect all the <tool>.sum files from the build tree.
46 5- Produce a report stating:
47 a- Failures expected in the manifest but not present in the build.
48 b- Failures in the build not expected in the manifest.
49 6- If all the build failures are expected in the manifest, it exits
50 with exit code 0. Otherwise, it exits with error code 1.
52 Manifest files contain expected DejaGNU results that are otherwise
54 They may also contain additional text:
56 # This is a comment. - self explanatory
57 @include file - the file is a path relative to the includer
58 @remove result text - result text is removed from the expected set
67 # Handled test results.
68 _VALID_TEST_RESULTS
= [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ]
69 _VALID_TEST_RESULTS_REX
= re
.compile("%s" % "|".join(_VALID_TEST_RESULTS
))
71 # Subdirectory of srcdir in which to find the manifest file.
72 _MANIFEST_SUBDIR
= 'contrib/testsuite-management'
74 # Pattern for naming manifest files.
75 # The first argument should be the toplevel GCC(/GNU tool) source directory.
76 # The second argument is the manifest subdir.
77 # The third argument is the manifest target, which defaults to the target
78 # triplet used during the build.
79 _MANIFEST_PATH_PATTERN
= '%s/%s/%s.xfail'
81 # The options passed to the program.
85 print >>sys
.stderr
, 'error: %s' % msg
89 class TestResult(object):
90 """Describes a single DejaGNU test result as emitted in .sum files.
92 We are only interested in representing unsuccessful tests. So, only
93 a subset of all the tests are loaded.
95 The summary line used to build the test result should have this format:
97 attrlist | XPASS: gcc.dg/unroll_1.c (test for excess errors)
98 ^^^^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^
99 optional state name description
103 attrlist: A comma separated list of attributes.
105 flaky Indicates that this test may not always fail. These
106 tests are reported, but their presence does not affect
109 expire=YYYYMMDD After this date, this test will produce an error
110 whether it is in the manifest or not.
112 state: One of UNRESOLVED, XPASS or FAIL.
113 name: File name for the test.
114 description: String describing the test (flags used, dejagnu message, etc)
115 ordinal: Monotonically increasing integer.
116 It is used to keep results for one .exp file sorted
117 by the order the tests were run.
120 def __init__(self
, summary_line
, ordinal
=-1):
123 if '|' in summary_line
:
124 (self
.attrs
, summary_line
) = summary_line
.split('|', 1)
128 self
.description
) = re
.match(r
' *([A-Z]+):\s*(\S+)\s+(.*)',
129 summary_line
).groups()
131 print 'Failed to parse summary line: "%s"' % summary_line
133 self
.attrs
= self
.attrs
.strip()
134 self
.state
= self
.state
.strip()
135 self
.description
= self
.description
.strip()
136 self
.ordinal
= ordinal
138 Error('Cannot parse summary line "%s"' % summary_line
)
140 if self
.state
not in _VALID_TEST_RESULTS
:
141 Error('Invalid test result %s in "%s" (parsed as "%s")' % (
142 self
.state
, summary_line
, self
))
144 def __lt__(self
, other
):
145 return (self
.name
< other
.name
or
146 (self
.name
== other
.name
and self
.ordinal
< other
.ordinal
))
149 return hash(self
.state
) ^
hash(self
.name
) ^
hash(self
.description
)
151 def __eq__(self
, other
):
152 return (self
.state
== other
.state
and
153 self
.name
== other
.name
and
154 self
.description
== other
.description
)
156 def __ne__(self
, other
):
157 return not (self
== other
)
162 attrs
= '%s | ' % self
.attrs
163 return '%s%s: %s %s' % (attrs
, self
.state
, self
.name
, self
.description
)
165 def ExpirationDate(self
):
166 # Return a datetime.date object with the expiration date for this
167 # test result. Return None, if no expiration has been set.
168 if re
.search(r
'expire=', self
.attrs
):
169 expiration
= re
.search(r
'expire=(\d\d\d\d)(\d\d)(\d\d)', self
.attrs
)
171 Error('Invalid expire= format in "%s". Must be of the form '
172 '"expire=YYYYMMDD"' % self
)
173 return datetime
.date(int(expiration
.group(1)),
174 int(expiration
.group(2)),
175 int(expiration
.group(3)))
178 def HasExpired(self
):
179 # Return True if the expiration date of this result has passed.
180 expiration_date
= self
.ExpirationDate()
182 now
= datetime
.date
.today()
183 return now
> expiration_date
186 def GetMakefileValue(makefile_name
, value_name
):
187 if os
.path
.exists(makefile_name
):
188 makefile
= open(makefile_name
)
189 for line
in makefile
:
190 if line
.startswith(value_name
):
191 (_
, value
) = line
.split('=', 1)
192 value
= value
.strip()
199 def ValidBuildDirectory(builddir
, target
):
200 if (not os
.path
.exists(builddir
) or
201 not os
.path
.exists('%s/Makefile' % builddir
) or
202 (not os
.path
.exists('%s/build-%s' % (builddir
, target
)) and
203 not os
.path
.exists('%s/%s' % (builddir
, target
)))):
209 """Return True if line is a comment."""
210 return line
.startswith('#')
213 def IsInterestingResult(line
):
214 """Return True if line is one of the summary lines we care about."""
216 (_
, line
) = line
.split('|', 1)
218 return bool(_VALID_TEST_RESULTS_REX
.match(line
))
222 """Return True if line is an include of another file."""
223 return line
.startswith("@include ")
226 def GetIncludeFile(line
, includer
):
227 """Extract the name of the include file from line."""
228 includer_dir
= os
.path
.dirname(includer
)
229 include_file
= line
[len("@include "):]
230 return os
.path
.join(includer_dir
, include_file
.strip())
233 def IsNegativeResult(line
):
234 """Return True if line should be removed from the expected results."""
235 return line
.startswith("@remove ")
238 def GetNegativeResult(line
):
239 """Extract the name of the negative result from line."""
240 line
= line
[len("@remove "):]
244 def ParseManifestWorker(result_set
, manifest_path
):
245 """Read manifest_path, adding the contents to result_set."""
246 if _OPTIONS
.verbosity
>= 1:
247 print 'Parsing manifest file %s.' % manifest_path
248 manifest_file
= open(manifest_path
)
249 for line
in manifest_file
:
253 elif IsComment(line
):
255 elif IsNegativeResult(line
):
256 result_set
.remove(TestResult(GetNegativeResult(line
)))
257 elif IsInclude(line
):
258 ParseManifestWorker(result_set
, GetIncludeFile(line
, manifest_path
))
259 elif IsInterestingResult(line
):
260 result_set
.add(TestResult(line
))
262 Error('Unrecognized line in manifest file: %s' % line
)
263 manifest_file
.close()
266 def ParseManifest(manifest_path
):
267 """Create a set of TestResult instances from the given manifest file."""
269 ParseManifestWorker(result_set
, manifest_path
)
273 def ParseSummary(sum_fname
):
274 """Create a set of TestResult instances from the given summary file."""
276 # ordinal is used when sorting the results so that tests within each
277 # .exp file are kept sorted.
279 sum_file
= open(sum_fname
)
280 for line
in sum_file
:
281 if IsInterestingResult(line
):
282 result
= TestResult(line
, ordinal
)
284 if result
.HasExpired():
285 # Tests that have expired are not added to the set of expected
286 # results. If they are still present in the set of actual results,
287 # they will cause an error to be reported.
288 print 'WARNING: Expected failure "%s" has expired.' % line
.strip()
290 result_set
.add(result
)
295 def GetManifest(manifest_path
):
296 """Build a set of expected failures from the manifest file.
298 Each entry in the manifest file should have the format understood
299 by the TestResult constructor.
301 If no manifest file exists for this target, it returns an empty set.
303 if os
.path
.exists(manifest_path
):
304 return ParseManifest(manifest_path
)
309 def CollectSumFiles(builddir
):
311 for root
, dirs
, files
in os
.walk(builddir
):
312 for ignored
in ('.svn', '.git'):
316 if fname
.endswith('.sum'):
317 sum_files
.append(os
.path
.join(root
, fname
))
321 def GetResults(sum_files
):
322 """Collect all the test results from the given .sum files."""
323 build_results
= set()
324 for sum_fname
in sum_files
:
325 print '\t%s' % sum_fname
326 build_results |
= ParseSummary(sum_fname
)
330 def CompareResults(manifest
, actual
):
331 """Compare sets of results and return two lists:
332 - List of results present in ACTUAL but missing from MANIFEST.
333 - List of results present in MANIFEST but missing from ACTUAL.
335 # Collect all the actual results not present in the manifest.
336 # Results in this set will be reported as errors.
337 actual_vs_manifest
= set()
338 for actual_result
in actual
:
339 if actual_result
not in manifest
:
340 actual_vs_manifest
.add(actual_result
)
342 # Collect all the tests in the manifest that were not found
343 # in the actual results.
344 # Results in this set will be reported as warnings (since
345 # they are expected failures that are not failing anymore).
346 manifest_vs_actual
= set()
347 for expected_result
in manifest
:
348 # Ignore tests marked flaky.
349 if 'flaky' in expected_result
.attrs
:
351 if expected_result
not in actual
:
352 manifest_vs_actual
.add(expected_result
)
354 return actual_vs_manifest
, manifest_vs_actual
357 def GetManifestPath(srcdir
, target
, user_provided_must_exist
):
358 """Return the full path to the manifest file."""
359 manifest_path
= _OPTIONS
.manifest
361 if user_provided_must_exist
and not os
.path
.exists(manifest_path
):
362 Error('Manifest does not exist: %s' % manifest_path
)
365 assert srcdir
and target
366 return _MANIFEST_PATH_PATTERN
% (srcdir
, _MANIFEST_SUBDIR
, target
)
370 srcdir
= GetMakefileValue('%s/Makefile' % _OPTIONS
.build_dir
, 'srcdir =')
371 target
= GetMakefileValue('%s/Makefile' % _OPTIONS
.build_dir
, 'target_alias=')
372 if not ValidBuildDirectory(_OPTIONS
.build_dir
, target
):
373 # If we have been given a set of results to use, we may
374 # not be inside a valid GCC build directory. In that case,
375 # the user must provide both a manifest file and a set
376 # of results to check against it.
377 if not _OPTIONS
.results
or not _OPTIONS
.manifest
:
378 Error('%s is not a valid GCC top level build directory. '
379 'You must use --manifest and --results to do the validation.' %
383 print 'Source directory: %s' % srcdir
384 print 'Build target: %s' % target
385 return srcdir
, target
388 def PrintSummary(msg
, summary
):
390 for result
in sorted(summary
):
394 def GetSumFiles(results
, build_dir
):
396 print 'Getting actual results from build directory %s' % build_dir
397 sum_files
= CollectSumFiles(build_dir
)
399 print 'Getting actual results from user-provided results'
400 sum_files
= results
.split()
404 def PerformComparison(expected
, actual
, ignore_missing_failures
):
405 actual_vs_expected
, expected_vs_actual
= CompareResults(expected
, actual
)
408 if len(actual_vs_expected
) > 0:
409 PrintSummary('Unexpected results in this build (new failures)',
413 if not ignore_missing_failures
and len(expected_vs_actual
) > 0:
414 PrintSummary('Expected results not present in this build (fixed tests)'
415 '\n\nNOTE: This is not a failure. It just means that these '
416 'tests were expected\nto fail, but they worked in this '
417 'configuration.\n', expected_vs_actual
)
420 print '\nSUCCESS: No unexpected failures.'
425 def CheckExpectedResults():
426 srcdir
, target
= GetBuildData()
427 manifest_path
= GetManifestPath(srcdir
, target
, True)
428 print 'Manifest: %s' % manifest_path
429 manifest
= GetManifest(manifest_path
)
430 sum_files
= GetSumFiles(_OPTIONS
.results
, _OPTIONS
.build_dir
)
431 actual
= GetResults(sum_files
)
433 if _OPTIONS
.verbosity
>= 1:
434 PrintSummary('Tests expected to fail', manifest
)
435 PrintSummary('\nActual test results', actual
)
437 return PerformComparison(manifest
, actual
, _OPTIONS
.ignore_missing_failures
)
440 def ProduceManifest():
441 (srcdir
, target
) = GetBuildData()
442 manifest_path
= GetManifestPath(srcdir
, target
, False)
443 print 'Manifest: %s' % manifest_path
444 if os
.path
.exists(manifest_path
) and not _OPTIONS
.force
:
445 Error('Manifest file %s already exists.\nUse --force to overwrite.' %
448 sum_files
= GetSumFiles(_OPTIONS
.results
, _OPTIONS
.build_dir
)
449 actual
= GetResults(sum_files
)
450 manifest_file
= open(manifest_path
, 'w')
451 for result
in sorted(actual
):
453 manifest_file
.write('%s\n' % result
)
454 manifest_file
.close()
460 (srcdir
, target
) = GetBuildData()
462 sum_files
= GetSumFiles(_OPTIONS
.results
, _OPTIONS
.build_dir
)
463 actual
= GetResults(sum_files
)
465 clean_sum_files
= GetSumFiles(_OPTIONS
.results
, _OPTIONS
.clean_build
)
466 clean
= GetResults(clean_sum_files
)
468 return PerformComparison(clean
, actual
, _OPTIONS
.ignore_missing_failures
)
472 parser
= optparse
.OptionParser(usage
=__doc__
)
474 # Keep the following list sorted by option name.
475 parser
.add_option('--build_dir', action
='store', type='string',
476 dest
='build_dir', default
='.',
477 help='Build directory to check (default = .)')
478 parser
.add_option('--clean_build', action
='store', type='string',
479 dest
='clean_build', default
=None,
480 help='Compare test results from this build against '
481 'those of another (clean) build. Use this option '
482 'when comparing the test results of your patch versus '
483 'the test results of a clean build without your patch. '
484 'You must provide the path to the top directory of your '
486 parser
.add_option('--force', action
='store_true', dest
='force',
487 default
=False, help='When used with --produce_manifest, '
488 'it will overwrite an existing manifest file '
490 parser
.add_option('--ignore_missing_failures', action
='store_true',
491 dest
='ignore_missing_failures', default
=False,
492 help='When a failure is expected in the manifest but '
493 'it is not found in the actual results, the script '
494 'produces a note alerting to this fact. This means '
495 'that the expected failure has been fixed, or '
496 'it did not run, or it may simply be flaky '
498 parser
.add_option('--manifest', action
='store', type='string',
499 dest
='manifest', default
=None,
500 help='Name of the manifest file to use (default = '
502 'contrib/testsuite-managment/<target_alias>.xfail)')
503 parser
.add_option('--produce_manifest', action
='store_true',
504 dest
='produce_manifest', default
=False,
505 help='Produce the manifest for the current '
506 'build (default = False)')
507 parser
.add_option('--results', action
='store', type='string',
508 dest
='results', default
=None, help='Space-separated list '
509 'of .sum files with the testing results to check. The '
510 'only content needed from these files are the lines '
511 'starting with FAIL, XPASS or UNRESOLVED (default = '
512 '.sum files collected from the build directory).')
513 parser
.add_option('--verbosity', action
='store', dest
='verbosity',
514 type='int', default
=0, help='Verbosity level (default = 0)')
516 (_OPTIONS
, _
) = parser
.parse_args(argv
[1:])
518 if _OPTIONS
.produce_manifest
:
519 retval
= ProduceManifest()
520 elif _OPTIONS
.clean_build
:
521 retval
= CompareBuilds()
523 retval
= CheckExpectedResults()
531 if __name__
== '__main__':
532 retval
= Main(sys
.argv
)