]> git.ipfire.org Git - thirdparty/git.git/blame - t/perf/aggregate.perl
perf aggregate: remove GIT_TEST_INSTALLED from --codespeed
[thirdparty/git.git] / t / perf / aggregate.perl
CommitLineData
342e9ef2
TR
1#!/usr/bin/perl
2
20d2a30f 3use lib '../../perl/build/lib';
342e9ef2
TR
4use strict;
5use warnings;
05eb1c37 6use JSON;
38368cba 7use Getopt::Long;
342e9ef2
TR
8use Git;
9
10sub get_times {
11 my $name = shift;
12 open my $fh, "<", $name or return undef;
13 my $line = <$fh>;
14 return undef if not defined $line;
15 close $fh or die "cannot close $name: $!";
22bec79d
JK
16 # times
17 if ($line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/) {
18 my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
19 return ($rt, $4, $5);
20 # size
21 } elsif ($line =~ /^\d+$/) {
22 return $&;
23 } else {
24 die "bad input line: $line";
25 }
342e9ef2
TR
26}
27
5a924a62
JK
28sub relative_change {
29 my ($r, $firstr) = @_;
30 if ($firstr > 0) {
31 return sprintf "%+.1f%%", 100.0*($r-$firstr)/$firstr;
32 } elsif ($r == 0) {
33 return "=";
34 } else {
35 return "+inf";
36 }
37}
38
342e9ef2
TR
39sub format_times {
40 my ($r, $u, $s, $firstr) = @_;
22bec79d 41 # no value means we did not finish the test
342e9ef2
TR
42 if (!defined $r) {
43 return "<missing>";
44 }
22bec79d
JK
45 # a single value means we have a size, not times
46 if (!defined $u) {
47 return format_size($r, $firstr);
48 }
49 # otherwise, we have real/user/system times
342e9ef2 50 my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
5a924a62 51 $out .= ' ' . relative_change($r, $firstr) if defined $firstr;
342e9ef2
TR
52 return $out;
53}
54
38368cba
CC
55sub usage {
56 print <<EOT;
57./aggregate.perl [options] [--] [<dir_or_rev>...] [--] [<test_script>...] >
58
59 Options:
60 --codespeed * Format output for Codespeed
61 --reponame <str> * Send given reponame to codespeed
62 --sort-by <str> * Sort output (only "regression" criteria is supported)
63 --subsection <str> * Use results from given subsection
64
65EOT
66 exit(1);
67}
68
22bec79d
JK
69sub human_size {
70 my $n = shift;
71 my @units = ('', qw(K M G));
72 while ($n > 900 && @units > 1) {
73 $n /= 1000;
74 shift @units;
75 }
76 return $n unless length $units[0];
77 return sprintf '%.1f%s', $n, $units[0];
78}
79
80sub format_size {
81 my ($size, $first) = @_;
82 # match the width of a time: 0.00(0.00+0.00)
83 my $out = sprintf '%15s', human_size($size);
84 $out .= ' ' . relative_change($size, $first) if defined $first;
85 return $out;
86}
87
cd5d4bf6 88my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
2e3efd06 89 $codespeed, $sortby, $subsection, $reponame);
38368cba
CC
90
91Getopt::Long::Configure qw/ require_order /;
92
93my $rc = GetOptions("codespeed" => \$codespeed,
94 "reponame=s" => \$reponame,
95 "sort-by=s" => \$sortby,
96 "subsection=s" => \$subsection);
97usage() unless $rc;
98
342e9ef2
TR
99while (scalar @ARGV) {
100 my $arg = $ARGV[0];
101 my $dir;
102 last if -f $arg or $arg eq "--";
103 if (! -d $arg) {
104 my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
105 $dir = "build/".$rev;
106 } else {
107 $arg =~ s{/*$}{};
108 $dir = $arg;
109 $dirabbrevs{$dir} = $dir;
110 }
111 push @dirs, $dir;
112 $dirnames{$dir} = $arg;
113 my $prefix = $dir;
114 $prefix =~ tr/^a-zA-Z0-9/_/c;
115 $prefixes{$dir} = $prefix . '.';
116 shift @ARGV;
117}
118
119if (not @dirs) {
120 @dirs = ('.');
121}
122$dirnames{'.'} = $dirabbrevs{'.'} = "this tree";
123$prefixes{'.'} = '';
124
125shift @ARGV if scalar @ARGV and $ARGV[0] eq "--";
126
127@tests = @ARGV;
128if (not @tests) {
129 @tests = glob "p????-*.sh";
130}
131
5d445f34 132my $resultsdir = "test-results";
cd5d4bf6
CC
133
134if (! $subsection and
135 exists $ENV{GIT_PERF_SUBSECTION} and
136 $ENV{GIT_PERF_SUBSECTION} ne "") {
137 $subsection = $ENV{GIT_PERF_SUBSECTION};
138}
139
140if ($subsection) {
141 $resultsdir .= "/" . $subsection;
5d445f34
CC
142}
143
342e9ef2
TR
144my @subtests;
145my %shorttests;
146for my $t (@tests) {
147 $t =~ s{(?:.*/)?(p(\d+)-[^/]+)\.sh$}{$1} or die "bad test name: $t";
148 my $n = $2;
5d445f34 149 my $fname = "$resultsdir/$t.subtests";
342e9ef2
TR
150 open my $fp, "<", $fname or die "cannot open $fname: $!";
151 for (<$fp>) {
152 chomp;
153 /^(\d+)$/ or die "malformed subtest line: $_";
154 push @subtests, "$t.$1";
155 $shorttests{"$t.$1"} = "$n.$1";
156 }
157 close $fp or die "cannot close $fname: $!";
158}
159
160sub read_descr {
161 my $name = shift;
162 open my $fh, "<", $name or return "<error reading description>";
db7ed0f2 163 binmode $fh, ":utf8" or die "PANIC on binmode: $!";
342e9ef2
TR
164 my $line = <$fh>;
165 close $fh or die "cannot close $name";
166 chomp $line;
167 return $line;
168}
169
342e9ef2
TR
170sub have_duplicate {
171 my %seen;
172 for (@_) {
173 return 1 if exists $seen{$_};
174 $seen{$_} = 1;
175 }
176 return 0;
177}
178sub have_slash {
179 for (@_) {
180 return 1 if m{/};
181 }
182 return 0;
183}
184
c94b6ac5
CC
185sub display_dir {
186 my ($d) = @_;
187 return exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d};
188}
189
30ffff6e
CC
190sub print_default_results {
191 my %descrs;
192 my $descrlen = 4; # "Test"
193 for my $t (@subtests) {
194 $descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
195 $descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
342e9ef2 196 }
342e9ef2 197
30ffff6e
CC
198 my %newdirabbrevs = %dirabbrevs;
199 while (!have_duplicate(values %newdirabbrevs)) {
200 %dirabbrevs = %newdirabbrevs;
201 last if !have_slash(values %dirabbrevs);
202 %newdirabbrevs = %dirabbrevs;
203 for (values %newdirabbrevs) {
204 s{^[^/]*/}{};
205 }
206 }
207
208 my %times;
209 my @colwidth = ((0)x@dirs);
342e9ef2 210 for my $i (0..$#dirs) {
c94b6ac5 211 my $w = length display_dir($dirs[$i]);
342e9ef2 212 $colwidth[$i] = $w if $w > $colwidth[$i];
342e9ef2 213 }
30ffff6e
CC
214 for my $t (@subtests) {
215 my $firstr;
216 for my $i (0..$#dirs) {
217 my $d = $dirs[$i];
22bec79d
JK
218 my $base = "$resultsdir/$prefixes{$d}$t";
219 $times{$prefixes{$d}.$t} = [];
220 foreach my $type (qw(times size)) {
221 if (-e "$base.$type") {
222 $times{$prefixes{$d}.$t} = [get_times("$base.$type")];
223 last;
224 }
225 }
30ffff6e
CC
226 my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
227 my $w = length format_times($r,$u,$s,$firstr);
228 $colwidth[$i] = $w if $w > $colwidth[$i];
229 $firstr = $r unless defined $firstr;
230 }
231 }
232 my $totalwidth = 3*@dirs+$descrlen;
233 $totalwidth += $_ for (@colwidth);
db7ed0f2 234
30ffff6e 235 printf "%-${descrlen}s", "Test";
342e9ef2 236 for my $i (0..$#dirs) {
c94b6ac5 237 printf " %-$colwidth[$i]s", display_dir($dirs[$i]);
342e9ef2
TR
238 }
239 print "\n";
30ffff6e
CC
240 print "-"x$totalwidth, "\n";
241 for my $t (@subtests) {
242 printf "%-${descrlen}s", $descrs{$t};
243 my $firstr;
244 for my $i (0..$#dirs) {
245 my $d = $dirs[$i];
246 my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
247 printf " %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
248 $firstr = $r unless defined $firstr;
249 }
250 print "\n";
251 }
342e9ef2 252}
30ffff6e 253
2e3efd06
CC
254sub print_sorted_results {
255 my ($sortby) = @_;
256
257 if ($sortby ne "regression") {
38368cba
CC
258 print "Only 'regression' is supported as '--sort-by' argument\n";
259 usage();
2e3efd06
CC
260 }
261
262 my @evolutions;
263 for my $t (@subtests) {
264 my ($prevr, $prevu, $prevs, $prevrev);
265 for my $i (0..$#dirs) {
266 my $d = $dirs[$i];
267 my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
268 if ($i > 0 and defined $r and defined $prevr and $prevr > 0) {
269 my $percent = 100.0 * ($r - $prevr) / $prevr;
270 push @evolutions, { "percent" => $percent,
271 "test" => $t,
272 "prevrev" => $prevrev,
273 "rev" => $d,
274 "prevr" => $prevr,
275 "r" => $r,
276 "prevu" => $prevu,
277 "u" => $u,
278 "prevs" => $prevs,
279 "s" => $s};
280 }
281 ($prevr, $prevu, $prevs, $prevrev) = ($r, $u, $s, $d);
282 }
283 }
284
285 my @sorted_evolutions = sort { $b->{percent} <=> $a->{percent} } @evolutions;
286
287 for my $e (@sorted_evolutions) {
288 printf "%+.1f%%", $e->{percent};
289 print " " . $e->{test};
290 print " " . format_times($e->{prevr}, $e->{prevu}, $e->{prevs});
291 print " " . format_times($e->{r}, $e->{u}, $e->{s});
292 print " " . display_dir($e->{prevrev});
293 print " " . display_dir($e->{rev});
294 print "\n";
295 }
296}
297
05eb1c37 298sub print_codespeed_results {
cd5d4bf6 299 my ($subsection) = @_;
05eb1c37
CC
300
301 my $project = "Git";
302
303 my $executable = `uname -s -m`;
304 chomp $executable;
305
cd5d4bf6
CC
306 if ($subsection) {
307 $executable .= ", " . $subsection;
05eb1c37
CC
308 }
309
310 my $environment;
fb2c362e
CC
311 if ($reponame) {
312 $environment = $reponame;
313 } elsif (exists $ENV{GIT_PERF_REPO_NAME} and $ENV{GIT_PERF_REPO_NAME} ne "") {
05eb1c37 314 $environment = $ENV{GIT_PERF_REPO_NAME};
05eb1c37
CC
315 } else {
316 $environment = `uname -r`;
317 chomp $environment;
318 }
319
320 my @data;
321
322 for my $t (@subtests) {
323 for my $d (@dirs) {
324 my $commitid = $prefixes{$d};
325 $commitid =~ s/^build_//;
326 $commitid =~ s/\.$//;
327 my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");
328
329 my %vals = (
330 "commitid" => $commitid,
331 "project" => $project,
332 "branch" => $dirnames{$d},
333 "executable" => $executable,
334 "benchmark" => $shorttests{$t} . " " . read_descr("$resultsdir/$t.descr"),
335 "environment" => $environment,
336 "result_value" => $result_value,
337 );
338 push @data, \%vals;
339 }
340 }
341
ed103edf 342 print to_json(\@data, {utf8 => 1, pretty => 1, canonical => 1}), "\n";
05eb1c37
CC
343}
344
30ffff6e
CC
345binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
346
05eb1c37 347if ($codespeed) {
cd5d4bf6 348 print_codespeed_results($subsection);
2e3efd06
CC
349} elsif (defined $sortby) {
350 print_sorted_results($sortby);
05eb1c37
CC
351} else {
352 print_default_results();
353}