--- /dev/null
+<!--
+Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+
+SPDX-License-Identifier: curl
+-->
+
+# scorecard.py
+
+This is an internal script in `tests/http/scorecard.py` used for testing
+curl's performance in a set of cases. These are for exercising parts of
+curl/libcurl in a reproducible fashion to judge improvements or detect
+regressions. They are not intended to represent real world scenarios
+as such.
+
+This script is not part of any official interface and we may
+change it in the future according to the project's needs.
+
+## setup
+
+When you are able to run curl's `pytest` suite, scorecard should work
+for you as well. They start a local Apache httpd or Caddy server and
+invoke the locally build `src/curl` (by default).
+
+## invocation
+
+A typical invocation for measuring performance of HTTP/2 downloads would be:
+
+```
+curl> python3 tests/http/scorecard.py -d h2
+```
+
+and this prints a table with the results. The last argument is the protocol to test and
+it can be `h1`, `h2` or `h3`. You can add `--json` to get results in JSON instead of text.
+
+Help for all command line options are available via:
+
+```
+curl> python3 tests/http/scorecard.py -h
+```
+
+## scenarios
+
+Apart from `-d/--downloads` there is `-u/--uploads` and `-r/--requests`. These are run with
+a variation of resource sizes and parallelism by default. You can specify these in some way
+if you are just interested in a particular case.
+
+For example, to run downloads of a 1 MB resource only, 100 times with at max 6 parallel transfers, use:
+
+```
+curl> python3 tests/http/scorecard.py -d --download-sizes=1mb --download-count=100 --download-parallel=6 h2
+```
+
+Similar options are available for uploads and requests scenarios.
+
+## dtrace
+
+With the `--dtrace` option, scorecard produces a dtrace sample of the user stacks in `tests/http/gen/curl/curl.user_stacks`. On many platforms, `dtrace` requires **special permissions**. It is therefore invoked via `sudo` and you should make sure that sudo works for the run without prompting for a password.
+
+Note: the file is the trace of the last curl invocation by scorecard. Use the parameters to narrow down the runs to the particular case you are interested in.
+
+## flame graphs
+
+With the excellent [Flame Graph](https://github.com/brendangregg/FlameGraph) by Brendan Gregg, scorecard can turn the `dtrace` samples into an interactive SVG. Set the environment variable `FLAMEGRAPH` to the location of your clone of that project and invoked scorecard with the `--flame` option. Like
+
+```
+curl> FLAMEGRAPH=/Users/sei/projects/FlameGraph python3 tests/http/scorecard.py \
+ -r --request-count=50000 --request-parallels=100 --samples=1 --flame h2
+```
+and the SVG of the run is in `tests/http/gen/curl/curl.flamegraph.svg`. You can open that in Firefox and zoom in/out of stacks of interest.
+
+Note: as with `dtrace`, the flame graph is for the last invocation of curl done by scorecard.
curl_verbose: int,
download_parallel: int = 0,
server_addr: Optional[str] = None,
- with_dtrace: bool = False):
+ with_dtrace: bool = False,
+ with_flame: bool = False):
self.verbose = verbose
self.env = env
self.protocol = protocol
self._silent_curl = not curl_verbose
self._download_parallel = download_parallel
self._with_dtrace = with_dtrace
+ self._with_flame = with_flame
def info(self, msg):
if self.verbose > 0:
def mk_curl_client(self):
return CurlClient(env=self.env, silent=self._silent_curl,
server_addr=self.server_addr,
- with_dtrace=self._with_dtrace)
+ with_dtrace=self._with_dtrace,
+ with_flame=self._with_flame)
def handshakes(self) -> Dict[str, Any]:
props = {}
verbose=args.verbose,
curl_verbose=args.curl_verbose,
download_parallel=args.download_parallel,
- with_dtrace=args.dtrace)
+ with_dtrace=args.dtrace,
+ with_flame=args.flame)
cards.append(card)
if test_httpd:
server_port=server_port,
verbose=args.verbose, curl_verbose=args.curl_verbose,
download_parallel=args.download_parallel,
- with_dtrace=args.dtrace)
+ with_dtrace=args.dtrace,
+ with_flame=args.flame)
card.setup_resources(server_docs, downloads)
cards.append(card)
parser.add_argument("--remote", action='store', type=str,
default=None, help="score against the remote server at <ip>:<port>")
parser.add_argument("--dtrace", action='store_true',
- default = False, help = "produce dtrace of curl")
+ default = False, help="produce dtrace of curl")
+ parser.add_argument("--flame", action='store_true',
+ default = False, help="produce a flame graph on curl, implies --dtrace")
parser.add_argument("-H", "--handshakes", action='store_true',
default=False, help="evaluate handshakes only")
self._pid = pid
self._run_dir = run_dir
self._proc = None
+ self._rc = 0
+ self._file = os.path.join(self._run_dir, 'curl.user_stacks')
def start(self):
+ if os.path.exists(self._file):
+ os.remove(self._file)
args = [
'sudo', 'dtrace',
'-x', 'ustackframes=100',
'-n', f'profile-97 /pid == {self._pid}/ {{ @[ustack()] = count(); }} tick-60s {{ exit(0); }}',
- '-o', f'{self._run_dir}/curl.user_stacks'
+ '-o', f'{self._file}'
]
self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False)
assert self._proc
def finish(self):
if self._proc:
self._proc.terminate()
+ self._rc = self._proc.returncode
+
+ @property
+ def file(self):
+ return self._file
class RunTcpDump:
silent: bool = False,
run_env: Optional[Dict[str, str]] = None,
server_addr: Optional[str] = None,
- with_dtrace: bool = False):
+ with_dtrace: bool = False,
+ with_flame: bool = False):
self.env = env
self._timeout = timeout if timeout else env.test_timeout
self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl
self._headerfile = f'{self._run_dir}/curl.headers'
self._log_path = f'{self._run_dir}/curl.log'
self._with_dtrace = with_dtrace
+ self._with_flame = with_flame
+ if self._with_flame:
+ self._with_dtrace = True
self._silent = silent
self._run_env = run_env
self._server_addr = server_addr if server_addr else '127.0.0.1'
exception = None
profile = None
tcpdump = None
- started_at = datetime.now()
+ dtrace = None
if with_tcpdump:
tcpdump = RunTcpDump(self.env, self._run_dir)
tcpdump.start()
+ started_at = datetime.now()
try:
with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr:
if with_profile:
ptimeout = 0.01
exitcode = p.returncode
profile.finish()
- if self._with_dtrace:
- dtrace.finish()
log.info(f'done: exit={exitcode}, profile={profile}')
else:
p = subprocess.run(args, stderr=cerr, stdout=cout,
f'(configured {self._timeout}s): {args}')
exitcode = -1
exception = 'TimeoutExpired'
+ ended_at = datetime.now()
if tcpdump:
tcpdump.finish()
+ if dtrace:
+ dtrace.finish()
+ if self._with_flame and dtrace:
+ self._generate_flame(dtrace, args)
coutput = open(self._stdoutfile).readlines()
cerrput = open(self._stderrfile).readlines()
return ExecResult(args=args, exit_code=exitcode, exception=exception,
stdout=coutput, stderr=cerrput,
- duration=datetime.now() - started_at,
+ duration=ended_at - started_at,
with_stats=with_stats,
profile=profile, tcpdump=tcpdump)
fin_response(response)
return r
+
+ def _generate_flame(self, dtrace: DTraceProfile, curl_args: List[str]):
+ log.info('generating flame graph from dtrace for this run')
+ if not os.path.exists(dtrace.file):
+ raise Exception(f'dtrace output file does not exist: {dtrace.file}')
+ if 'FLAMEGRAPH' not in os.environ:
+ raise Exception('Env variable FLAMEGRAPH not set')
+ fg_dir = os.environ['FLAMEGRAPH']
+ if not os.path.exists(fg_dir):
+ raise Exception(f'FlameGraph directory not found: {fg_dir}')
+
+ fg_collapse = os.path.join(fg_dir, 'stackcollapse.pl')
+ if not os.path.exists(fg_collapse):
+ raise Exception(f'FlameGraph script not found: {fg_collapse}')
+
+ fg_gen_flame = os.path.join(fg_dir, 'flamegraph.pl')
+ if not os.path.exists(fg_gen_flame):
+ raise Exception(f'FlameGraph script not found: {fg_gen_flame}')
+
+ file_collapsed = f'{dtrace.file}.collapsed'
+ file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg')
+ file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr')
+ log.info('waiting a sec for dtrace to finish flusheing its buffers')
+ time.sleep(1)
+ log.info(f'collapsing stacks into {file_collapsed}')
+ with open(file_collapsed, 'w') as cout, open(file_err, 'w') as cerr:
+ p = subprocess.run([
+ fg_collapse, dtrace.file
+ ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+ rc = p.returncode
+ if rc != 0:
+ raise Exception(f'{fg_collapse} returned error {rc}')
+ log.info(f'generating graph into {file_svg}')
+ cmdline = ' '.join(curl_args)
+ if len(cmdline) > 80:
+ title = f'{cmdline[:80]}...'
+ subtitle = f'...{cmdline[-80:]}'
+ else:
+ title = cmdline
+ subtitle = ''
+ with open(file_svg, 'w') as cout, open(file_err, 'w') as cerr:
+ p = subprocess.run([
+ fg_gen_flame, '--colors', 'green',
+ '--title', title, '--subtitle', subtitle,
+ file_collapsed
+ ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+ rc = p.returncode
+ if rc != 0:
+ raise Exception(f'{fg_gen_flame} returned error {rc}')
projects / thirdparty / curl.git / commitdiff
