[thirdparty/gcc.git] / contrib / analyze_brprob.py

#!/usr/bin/env python3
#
# Script to analyze results of our branch prediction heuristics
#
# This file is part of GCC.
#
# GCC is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3, or (at your option) any later
# version.
#
# GCC is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with GCC; see the file COPYING3.  If not see
# <http://www.gnu.org/licenses/>.  */
#
#
#
# This script is used to calculate two basic properties of the branch prediction
# heuristics - coverage and hitrate.  Coverage is number of executions
# of a given branch matched by the heuristics and hitrate is probability
# that once branch is predicted as taken it is really taken.
#
# These values are useful to determine the quality of given heuristics.
# Hitrate may be directly used in predict.def.
#
# Usage:
#  Step 1: Compile and profile your program.  You need to use -fprofile-generate
#    flag to get the profiles.
#  Step 2: Make a reference run of the intrumented application.
#  Step 3: Compile the program with collected profile and dump IPA profiles
#          (-fprofile-use -fdump-ipa-profile-details)
#  Step 4: Collect all generated dump files:
#          find . -name '*.profile' | xargs cat > dump_file
#  Step 5: Run the script:
#          ./analyze_brprob.py dump_file
#          and read results.  Basically the following table is printed:
#
# HEURISTICS                           BRANCHES  (REL)  HITRATE                COVERAGE  (REL)
# early return (on trees)                     3   0.2%  35.83% /  93.64%          66360   0.0%
# guess loop iv compare                       8   0.6%  53.35% /  53.73%       11183344   0.0%
# call                                       18   1.4%  31.95% /  69.95%       51880179   0.2%
# loop guard                                 23   1.8%  84.13% /  84.85%    13749065956  42.2%
# opcode values positive (on trees)          42   3.3%  15.71% /  84.81%     6771097902  20.8%
# opcode values nonequal (on trees)         226  17.6%  72.48% /  72.84%      844753864   2.6%
# loop exit                                 231  18.0%  86.97% /  86.98%     8952666897  27.5%
# loop iterations                           239  18.6%  91.10% /  91.10%     3062707264   9.4%
# DS theory                                 281  21.9%  82.08% /  83.39%     7787264075  23.9%
# no prediction                             293  22.9%  46.92% /  70.70%     2293267840   7.0%
# guessed loop iterations                   313  24.4%  76.41% /  76.41%    10782750177  33.1%
# first match                               708  55.2%  82.30% /  82.31%    22489588691  69.0%
# combined                                 1282 100.0%  79.76% /  81.75%    32570120606 100.0%
#
#
#  The heuristics called "first match" is a heuristics used by GCC branch
#  prediction pass and it predicts 55.2% branches correctly. As you can,
#  the heuristics has very good covertage (69.05%).  On the other hand,
#  "opcode values nonequal (on trees)" heuristics has good hirate, but poor
#  coverage.

import sys
import os
import re
import argparse

from math import *

counter_aggregates = set(['combined', 'first match', 'DS theory',
    'no prediction'])

def percentage(a, b):
    return 100.0 * a / b

def average(values):
    return 1.0 * sum(values) / len(values)

def average_cutoff(values, cut):
    l = len(values)
    skip = floor(l * cut / 2)
    if skip > 0:
        values.sort()
        values = values[skip:-skip]
    return average(values)

def median(values):
    values.sort()
    return values[int(len(values) / 2)]

class Summary:
    def __init__(self, name):
        self.name = name
        self.branches = 0
        self.successfull_branches = 0
        self.count = 0
        self.hits = 0
        self.fits = 0

    def get_hitrate(self):
        return 100.0 * self.hits / self.count

    def get_branch_hitrate(self):
        return 100.0 * self.successfull_branches / self.branches

    def count_formatted(self):
        v = self.count
        for unit in ['','K','M','G','T','P','E','Z']:
            if v < 1000:
                return "%3.2f%s" % (v, unit)
            v /= 1000.0
        return "%.1f%s" % (v, 'Y')

    def print(self, branches_max, count_max):
        print('%-40s %8i %5.1f%% %11.2f%% %7.2f%% / %6.2f%% %14i %8s %5.1f%%' %
            (self.name, self.branches,
                percentage(self.branches, branches_max),
                self.get_branch_hitrate(),
                self.get_hitrate(),
                percentage(self.fits, self.count),
                self.count, self.count_formatted(),
                percentage(self.count, count_max)))

class Profile:
    def __init__(self, filename):
        self.filename = filename
        self.heuristics = {}
        self.niter_vector = []

    def add(self, name, prediction, count, hits):
        if not name in self.heuristics:
            self.heuristics[name] = Summary(name)

        s = self.heuristics[name]
        s.branches += 1

        s.count += count
        if prediction < 50:
            hits = count - hits
        remaining = count - hits
        if hits >= remaining:
            s.successfull_branches += 1

        s.hits += hits
        s.fits += max(hits, remaining)

    def add_loop_niter(self, niter):
        if niter > 0:
            self.niter_vector.append(niter)

    def branches_max(self):
        return max([v.branches for k, v in self.heuristics.items()])

    def count_max(self):
        return max([v.count for k, v in self.heuristics.items()])

    def print_group(self, sorting, group_name, heuristics):
        count_max = self.count_max()
        branches_max = self.branches_max()

        sorter = lambda x: x.branches
        if sorting == 'branch-hitrate':
            sorter = lambda x: x.get_branch_hitrate()
        elif sorting == 'hitrate':
            sorter = lambda x: x.get_hitrate()
        elif sorting == 'coverage':
            sorter = lambda x: x.count
        elif sorting == 'name':
            sorter = lambda x: x.name.lower()

        print('%-40s %8s %6s %12s %18s %14s %8s %6s' %
            ('HEURISTICS', 'BRANCHES', '(REL)',
            'BR. HITRATE', 'HITRATE', 'COVERAGE', 'COVERAGE', '(REL)'))
        for h in sorted(heuristics, key = sorter):
            h.print(branches_max, count_max)

    def dump(self, sorting):
        heuristics = self.heuristics.values()
        if len(heuristics) == 0:
            print('No heuristics available')
            return

        special = list(filter(lambda x: x.name in counter_aggregates,
            heuristics))
        normal = list(filter(lambda x: x.name not in counter_aggregates,
            heuristics))

        self.print_group(sorting, 'HEURISTICS', normal)
        print()
        self.print_group(sorting, 'HEURISTIC AGGREGATES', special)

        if len(self.niter_vector) > 0:
            print ('\nLoop count: %d' % len(self.niter_vector)),
            print('  avg. # of iter: %.2f' % average(self.niter_vector))
            print('  median # of iter: %.2f' % median(self.niter_vector))
            for v in [1, 5, 10, 20, 30]:
                cut = 0.01 * v
                print('  avg. (%d%% cutoff) # of iter: %.2f'
                    % (v, average_cutoff(self.niter_vector, cut)))

parser = argparse.ArgumentParser()
parser.add_argument('dump_file', metavar = 'dump_file',
    help = 'IPA profile dump file')
parser.add_argument('-s', '--sorting', dest = 'sorting',
    choices = ['branches', 'branch-hitrate', 'hitrate', 'coverage', 'name'],
    default = 'branches')

args = parser.parse_args()

profile = Profile(sys.argv[1])
r = re.compile('  (.*) heuristics( of edge [0-9]*->[0-9]*)?( \\(.*\\))?: (.*)%.*exec ([0-9]*) hit ([0-9]*)')
loop_niter_str = ';;  profile-based iteration count: '
for l in open(args.dump_file).readlines():
    m = r.match(l)
    if m != None and m.group(3) == None:
        name = m.group(1)
        prediction = float(m.group(4))
        count = int(m.group(5))
        hits = int(m.group(6))

        profile.add(name, prediction, count, hits)
    elif l.startswith(loop_niter_str):
        v = int(l[len(loop_niter_str):])
        profile.add_loop_niter(v)

profile.dump(args.sorting)
Commit	Line	Data
4877829b ML	1	#!/usr/bin/env python3
	2	#
	3	# Script to analyze results of our branch prediction heuristics
	4	#
	5	# This file is part of GCC.
	6	#
	7	# GCC is free software; you can redistribute it and/or modify it under
	8	# the terms of the GNU General Public License as published by the Free
	9	# Software Foundation; either version 3, or (at your option) any later
	10	# version.
	11	#
	12	# GCC is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or
	14	# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	15	# for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with GCC; see the file COPYING3. If not see
	19	# <http://www.gnu.org/licenses/>. */
	20	#
	21	#
	22	#
	23	# This script is used to calculate two basic properties of the branch prediction
	24	# heuristics - coverage and hitrate. Coverage is number of executions
	25	# of a given branch matched by the heuristics and hitrate is probability
	26	# that once branch is predicted as taken it is really taken.
	27	#
	28	# These values are useful to determine the quality of given heuristics.
	29	# Hitrate may be directly used in predict.def.
	30	#
	31	# Usage:
	32	# Step 1: Compile and profile your program. You need to use -fprofile-generate
	33	# flag to get the profiles.
	34	# Step 2: Make a reference run of the intrumented application.
	35	# Step 3: Compile the program with collected profile and dump IPA profiles
	36	# (-fprofile-use -fdump-ipa-profile-details)
	37	# Step 4: Collect all generated dump files:
	38	# find . -name '*.profile' \| xargs cat > dump_file
	39	# Step 5: Run the script:
	40	# ./analyze_brprob.py dump_file
	41	# and read results. Basically the following table is printed:
	42	#
	43	# HEURISTICS BRANCHES (REL) HITRATE COVERAGE (REL)
	44	# early return (on trees) 3 0.2% 35.83% / 93.64% 66360 0.0%
	45	# guess loop iv compare 8 0.6% 53.35% / 53.73% 11183344 0.0%
	46	# call 18 1.4% 31.95% / 69.95% 51880179 0.2%
	47	# loop guard 23 1.8% 84.13% / 84.85% 13749065956 42.2%
	48	# opcode values positive (on trees) 42 3.3% 15.71% / 84.81% 6771097902 20.8%
	49	# opcode values nonequal (on trees) 226 17.6% 72.48% / 72.84% 844753864 2.6%
	50	# loop exit 231 18.0% 86.97% / 86.98% 8952666897 27.5%
	51	# loop iterations 239 18.6% 91.10% / 91.10% 3062707264 9.4%
	52	# DS theory 281 21.9% 82.08% / 83.39% 7787264075 23.9%
	53	# no prediction 293 22.9% 46.92% / 70.70% 2293267840 7.0%
	54	# guessed loop iterations 313 24.4% 76.41% / 76.41% 10782750177 33.1%
	55	# first match 708 55.2% 82.30% / 82.31% 22489588691 69.0%
	56	# combined 1282 100.0% 79.76% / 81.75% 32570120606 100.0%
	57	#
	58	#
	59	# The heuristics called "first match" is a heuristics used by GCC branch
	60	# prediction pass and it predicts 55.2% branches correctly. As you can,
	61	# the heuristics has very good covertage (69.05%). On the other hand,
	62	# "opcode values nonequal (on trees)" heuristics has good hirate, but poor
	63	# coverage.
	64
65	import sys
66	import os
67	import re
0d73e480	68	import argparse
4877829b	69
199b1891 ML	70	from math import *
199b1891 ML	71
ca3b6071 ML	72	counter_aggregates = set(['combined', 'first match', 'DS theory',
	73	'no prediction'])
	74
4877829b ML	75	def percentage(a, b):
	76	return 100.0 * a / b
	77
199b1891 ML	78	def average(values):
	79	return 1.0 * sum(values) / len(values)
	80
	81	def average_cutoff(values, cut):
	82	l = len(values)
	83	skip = floor(l * cut / 2)
	84	if skip > 0:
	85	values.sort()
	86	values = values[skip:-skip]
	87	return average(values)
	88
	89	def median(values):
	90	values.sort()
	91	return values[int(len(values) / 2)]
	92
4877829b ML	93	class Summary:
	94	def __init__(self, name):
	95	self.name = name
	96	self.branches = 0
ca3b6071	97	self.successfull_branches = 0
4877829b ML	98	self.count = 0
	99	self.hits = 0
	100	self.fits = 0
	101
0d73e480	102	def get_hitrate(self):
ca3b6071 ML	103	return 100.0 * self.hits / self.count
	104
	105	def get_branch_hitrate(self):
	106	return 100.0 * self.successfull_branches / self.branches
0d73e480	107
4877829b ML	108	def count_formatted(self):
	109	v = self.count
	110	for unit in ['','K','M','G','T','P','E','Z']:
	111	if v < 1000:
	112	return "%3.2f%s" % (v, unit)
	113	v /= 1000.0
	114	return "%.1f%s" % (v, 'Y')
	115
ca3b6071 ML	116	def print(self, branches_max, count_max):
	117	print('%-40s %8i %5.1f%% %11.2f%% %7.2f%% / %6.2f%% %14i %8s %5.1f%%' %
	118	(self.name, self.branches,
	119	percentage(self.branches, branches_max),
	120	self.get_branch_hitrate(),
	121	self.get_hitrate(),
	122	percentage(self.fits, self.count),
	123	self.count, self.count_formatted(),
	124	percentage(self.count, count_max)))
	125
4877829b ML	126	class Profile:
	127	def __init__(self, filename):
	128	self.filename = filename
	129	self.heuristics = {}
199b1891	130	self.niter_vector = []
4877829b ML	131
	132	def add(self, name, prediction, count, hits):
	133	if not name in self.heuristics:
	134	self.heuristics[name] = Summary(name)
	135
	136	s = self.heuristics[name]
	137	s.branches += 1
ca3b6071	138
4877829b ML	139	s.count += count
	140	if prediction < 50:
	141	hits = count - hits
ca3b6071 ML	142	remaining = count - hits
	143	if hits >= remaining:
	144	s.successfull_branches += 1
	145
4877829b	146	s.hits += hits
ca3b6071	147	s.fits += max(hits, remaining)
4877829b	148
199b1891 ML	149	def add_loop_niter(self, niter):
	150	if niter > 0:
	151	self.niter_vector.append(niter)
	152
4877829b ML	153	def branches_max(self):
	154	return max([v.branches for k, v in self.heuristics.items()])
	155
	156	def count_max(self):
	157	return max([v.count for k, v in self.heuristics.items()])
	158
ca3b6071 ML	159	def print_group(self, sorting, group_name, heuristics):
	160	count_max = self.count_max()
	161	branches_max = self.branches_max()
	162
	163	sorter = lambda x: x.branches
	164	if sorting == 'branch-hitrate':
	165	sorter = lambda x: x.get_branch_hitrate()
	166	elif sorting == 'hitrate':
	167	sorter = lambda x: x.get_hitrate()
0d73e480	168	elif sorting == 'coverage':
ca3b6071 ML	169	sorter = lambda x: x.count
	170	elif sorting == 'name':
	171	sorter = lambda x: x.name.lower()
	172
	173	print('%-40s %8s %6s %12s %18s %14s %8s %6s' %
	174	('HEURISTICS', 'BRANCHES', '(REL)',
	175	'BR. HITRATE', 'HITRATE', 'COVERAGE', 'COVERAGE', '(REL)'))
	176	for h in sorted(heuristics, key = sorter):
	177	h.print(branches_max, count_max)
	178
	179	def dump(self, sorting):
	180	heuristics = self.heuristics.values()
	181	if len(heuristics) == 0:
	182	print('No heuristics available')
	183	return
	184
	185	special = list(filter(lambda x: x.name in counter_aggregates,
	186	heuristics))
	187	normal = list(filter(lambda x: x.name not in counter_aggregates,
	188	heuristics))
0d73e480	189
ca3b6071 ML	190	self.print_group(sorting, 'HEURISTICS', normal)
	191	print()
	192	self.print_group(sorting, 'HEURISTIC AGGREGATES', special)
4877829b	193
88617fe4 ML	194	if len(self.niter_vector) > 0:
	195	print ('\nLoop count: %d' % len(self.niter_vector)),
	196	print(' avg. # of iter: %.2f' % average(self.niter_vector))
	197	print(' median # of iter: %.2f' % median(self.niter_vector))
	198	for v in [1, 5, 10, 20, 30]:
	199	cut = 0.01 * v
ca3b6071 ML	200	print(' avg. (%d%% cutoff) # of iter: %.2f'
ca3b6071 ML	201	% (v, average_cutoff(self.niter_vector, cut)))
199b1891	202
0d73e480	203	parser = argparse.ArgumentParser()
ca3b6071 ML	204	parser.add_argument('dump_file', metavar = 'dump_file',
	205	help = 'IPA profile dump file')
	206	parser.add_argument('-s', '--sorting', dest = 'sorting',
	207	choices = ['branches', 'branch-hitrate', 'hitrate', 'coverage', 'name'],
	208	default = 'branches')
0d73e480 ML	209
0d73e480 ML	210	args = parser.parse_args()
4877829b ML	211
4877829b ML	212	profile = Profile(sys.argv[1])
e49efc14	213	r = re.compile(' (.) heuristics( of edge [0-9]->[0-9])?( \\(.\\))?: (.)%.exec ([0-9]) hit ([0-9])')
199b1891	214	loop_niter_str = ';; profile-based iteration count: '
0d73e480	215	for l in open(args.dump_file).readlines():
4877829b	216	m = r.match(l)
e49efc14	217	if m != None and m.group(3) == None:
4877829b	218	name = m.group(1)
e49efc14 ML	219	prediction = float(m.group(4))
	220	count = int(m.group(5))
	221	hits = int(m.group(6))
4877829b ML	222
4877829b ML	223	profile.add(name, prediction, count, hits)
199b1891 ML	224	elif l.startswith(loop_niter_str):
	225	v = int(l[len(loop_niter_str):])
	226	profile.add_loop_niter(v)
4877829b	227
0d73e480	228	profile.dump(args.sorting)