]>
Commit | Line | Data |
---|---|---|
ed5ce4e0 MS |
1 | #!/usr/bin/perl |
2 | ||
3 | # Three-way DejaGNU comparison; uses dglib.pm. Run perldoc on this file for | |
4 | # usage. | |
5 | # | |
6 | # Author: Matthew Sachs <msachs@apple.com> | |
7 | # | |
8 | # Copyright (c) 2006 Free Software Foundation. | |
9 | # | |
10 | # This file is part of GCC. | |
11 | # | |
12 | # GCC is free software; you can redistribute it and/or modify | |
13 | # it under the terms of the GNU General Public License as published by | |
3dfb41c5 | 14 | # the Free Software Foundation; either version 3, or (at your option) |
ed5ce4e0 MS |
15 | # any later version. |
16 | # | |
17 | # GCC is distributed in the hope that it will be useful, | |
18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
20 | # GNU General Public License for more details. | |
21 | # | |
22 | # You should have received a copy of the GNU General Public License | |
23 | # along with GCC; see the file COPYING. If not, write to | |
24 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, | |
25 | # Boston, MA 02110-1301, USA. | |
26 | ||
27 | =pod | |
28 | ||
29 | =head1 SYNOPSIS | |
30 | ||
31 | compareSumTests3 -- Two-way or three-way compare between DejaGNU .sum files | |
32 | ||
33 | =head1 USAGE | |
34 | ||
35 | compareSumTests3 old1.sum [old2.sum] new.sum | |
36 | compareSumTests3 -i 1:2 -x 2:3 old1.sum old2.sum new.sum | |
37 | ||
38 | =head1 DESCRIPTION | |
39 | ||
40 | Gives results in terms of 'new' (e.g. things that work in 'new' and don't in | |
41 | other compilers are improvements, things that don't in 'new' and do in others | |
42 | are regressions, and it tells you which of the two old compilers (or both) | |
43 | the test is a regression from. | |
44 | ||
45 | We treat any DG result other than PASS or XFAIL as a failure, e.g. | |
46 | UNRESOLVED, UNTESTED or test was not run. | |
47 | ||
48 | We merge some tests into 'logical tests' with multiple subphases. | |
49 | For instance, some tests will have compile, execute, and link | |
50 | subtests. For these tests, if one of the phases fails, we | |
51 | indicate which phase the failure originates in. For instance, | |
52 | in the following test results: | |
53 | ||
54 | gcc.c-torture/compile_execute/xxxx.c: [FAIL:C,FAIL:X,PASS] | |
55 | ||
56 | the "compile_execute" replaces the compile or execute portion of the test name, | |
57 | and "FAIL:C" and "FAIL:X" indicates where the combined test failed. | |
58 | ||
59 | =head1 OPTIONS | |
60 | ||
61 | =head2 OVERVIEW | |
62 | ||
63 | =over 4 | |
64 | ||
65 | =item * | |
66 | ||
67 | C<-i X:Y>: Only display differences between the two indicated runs. | |
68 | ||
69 | =item * | |
70 | ||
71 | C<-p>: Give plain output, suitable for piping to another program. | |
72 | ||
73 | =item * | |
74 | ||
75 | C<-x X:Y>: Exclude differences between the two indicated runs. | |
76 | ||
77 | =back | |
78 | ||
79 | =head2 PLAIN OUTPUT FORMAT | |
80 | ||
81 | In the plain | |
82 | output format, the category headers are not displayed and there are no tabs | |
83 | in front of each result line. Instead, each result line has two characters | |
84 | followed by a space in front of it. The first character will be either an 'I' | |
85 | for improvement or 'R' for regression; the second character will be a 1, 2, or 3, | |
86 | indicating which run was the odd one out. | |
87 | ||
ed5ce4e0 MS |
88 | =head2 SELECTING CHANGE SUBSETS |
89 | ||
90 | The following options cause only a selected subset of changes to be displayed. | |
91 | These options ask for a "run", a number which is used to select | |
92 | one of the three runs (C<old1>, C<old2>, or C<new>.) C<1> and C<2> signify C<old1> and C<old2> | |
93 | respectively; 3 signifies C<new>. If multiple options are given, the changes displayed | |
94 | will be those which obey all of the given restrictions. | |
95 | ||
96 | Typical usage of these options is to express something like "give me all changes | |
97 | between 2 and 3, except for those where there was the same difference betwen 1 and 2 | |
98 | (as between 2 and 3.)" This would be given as: | |
99 | ||
100 | -i 2:3 -x 1:2 | |
101 | ||
102 | =over 4 | |
103 | ||
104 | =item * | |
105 | ||
106 | C<-i X:Y>: Only differences which are present between the two runs given | |
107 | are displayed. For instance, if C<-i 1:2> is given and test A passes in | |
108 | runs 1 and 2 but fails in run 3, that result will not be displayed. | |
109 | ||
110 | =item * | |
111 | ||
112 | C<-x X:Y>: Differences which are identical to a difference between the two runs | |
113 | given will B<not> be displayed. For instance, if C<-x 1:2> is given and | |
114 | test A passes in run 1 and fails in runs 2 and 3, that result will not be | |
115 | displayed (since C<-x> will cause the difference between 1 and 2 to be ignored, | |
116 | and the difference in 1 and 3 parallels the difference between 1 and 2.) | |
117 | This option may only be used in conjunction with C<-i>. | |
118 | ||
119 | =back | |
120 | ||
121 | =cut | |
122 | ||
123 | use strict; | |
124 | use warnings; | |
125 | use Getopt::Long; | |
126 | ||
127 | use FindBin qw($Bin); | |
128 | use lib "$Bin"; | |
129 | use dglib; | |
130 | ||
131 | my %options; | |
132 | my $error = undef; | |
133 | ||
134 | if(!GetOptions( | |
135 | "p" => \$options{p}, | |
136 | "i=s" => \$options{i}, | |
137 | "x=s" => \$options{x}, | |
138 | )) { | |
139 | $error = ""; | |
140 | } elsif(@ARGV != 2 and @ARGV != 3) { | |
141 | $error = ""; | |
142 | } elsif($options{x} and !$options{i}) { | |
143 | $error = "-x may only be given in conjunction with -i."; | |
144 | } else { | |
145 | foreach my $opt("i", "x") { | |
146 | if($options{$opt} and | |
147 | ($options{$opt} !~ /^([123]):([123])$/ or | |
148 | $1 == $2) | |
149 | ) { | |
150 | $error = "Invalid -$opt argument."; | |
151 | } | |
152 | } | |
153 | } | |
154 | ||
155 | if(defined($error)) { | |
156 | print STDERR "$error\n" if $error; | |
157 | print STDERR "Usage: compareSumTests3 [-p] [-i X:Y [-x X:Y]] old1.sum old2.sum new.sum\n"; | |
158 | print STDERR "Try 'perldoc $0' for further information.\n"; | |
159 | exit 1; | |
160 | } | |
161 | ||
162 | my(@sumfiles) = @ARGV; | |
163 | -f $_ || die "$_ is not a regular file!\n" foreach @sumfiles; | |
164 | my(%results, @inc_changes, @exc_changes, %checksums); | |
165 | ||
166 | # We decrement the values given so that they correspond | |
167 | # to indices into our results array. | |
168 | if($options{i}) { | |
169 | $options{i} =~ /(\d+):(\d+)/; | |
170 | @inc_changes = ($1 - 1, $2 - 1); | |
171 | } | |
172 | if($options{x}) { | |
173 | $options{x} =~ /(\d+):(\d+)/; | |
174 | @exc_changes = ($1 - 1, $2 - 1); | |
175 | } | |
176 | ||
177 | ||
178 | my %analyzed_results = compareSumFiles(\@sumfiles); | |
179 | ||
180 | foreach my $cat (qw(improvements regressions miscellaneous)) { | |
181 | if(@sumfiles == 3) { | |
182 | my @subcounts; | |
183 | if(!$options{p}) { | |
184 | $subcounts[$_] = @{$analyzed_results{$cat}->[$_] || []} for(0..2); | |
185 | print "\u$cat: ", ($subcounts[0]+$subcounts[1]+$subcounts[2]), "\n"; | |
186 | } | |
187 | ||
188 | for(my $i = 0; $i < 3; $i++) { | |
189 | if(!$options{p} and $cat ne "miscellaneous") { | |
190 | if($i == 0) { | |
191 | if($cat eq "regressions") { | |
192 | print "\tSuccess in old1 only: $subcounts[$i]\n"; | |
193 | } else { | |
194 | print "\tFailure in old1 only: $subcounts[$i]\n"; | |
195 | } | |
196 | } elsif($i == 1) { | |
197 | if($cat eq "regressions") { | |
198 | print "\tSuccess in old2 only: $subcounts[$i]\n"; | |
199 | } else { | |
200 | print "\tFailure in old2 only: $subcounts[$i]\n"; | |
201 | } | |
202 | } else { | |
203 | if($cat eq "regressions") { | |
204 | print "\tFailure in new only: $subcounts[$i]\n"; | |
205 | } else { | |
206 | print "\tSuccess in new only: $subcounts[$i]\n"; | |
207 | } | |
208 | } | |
209 | } | |
210 | ||
211 | foreach my $test (sort {$a->{name} cmp $b->{name}} @{$analyzed_results{$cat}->[$i] || []}) { | |
212 | if(!$options{p}) { | |
213 | if($cat eq "miscellaneous") { | |
214 | print "\t"; | |
215 | } else { | |
216 | print "\t\t"; | |
217 | } | |
218 | } else { | |
219 | if($cat eq "regressions") { | |
220 | print "R"; | |
221 | } else { | |
222 | print "I"; | |
223 | } | |
224 | ||
225 | print $i+1, " "; | |
226 | } | |
227 | printf "%s [%s,%s,%s]\n", $test->{name}, $test->{data}->[0], $test->{data}->[1], $test->{data}->[2]; | |
228 | } | |
229 | } | |
230 | } else { | |
231 | if(!$options{p}) { | |
232 | my $subcount = @{$analyzed_results{$cat}}; | |
233 | print "\u$cat: $subcount\n"; | |
234 | } | |
235 | ||
236 | foreach my $test (sort {$a->{name} cmp $b->{name}} @{$analyzed_results{$cat}}) { | |
237 | if(!$options{p}) { | |
238 | print "\t"; | |
239 | } else { | |
240 | if($cat eq "regressions") { | |
241 | print "R"; } else { | |
242 | print "I"; | |
243 | } | |
244 | ||
245 | print " "; | |
246 | } | |
247 | printf "%s [%s,%s]\n", $test->{name}, $test->{data}->[0], $test->{data}->[1], $test->{data}->[2]; | |
248 | } | |
249 | } | |
250 | } |