From: Stefan Eissing Date: Tue, 1 Jul 2025 10:19:28 +0000 (+0200) Subject: scorecard: flame graphs and documentation X-Git-Tag: rc-8_15_0-3~28 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=51f933801eb24a99e347c2833f2bd00928bf5dbd;p=thirdparty%2Fcurl.git scorecard: flame graphs and documentation Add `--flame` option to scorecard.py for generating flame graphs. Add documentation in docs/internal/SCORECARD.md on how to use this. Closes #17792 --- diff --git a/.github/scripts/spellcheck.words b/.github/scripts/spellcheck.words index 1d44eb9625..a1d030dd83 100644 --- a/.github/scripts/spellcheck.words +++ b/.github/scripts/spellcheck.words @@ -205,6 +205,7 @@ DoT doxygen drftpd dsa +dtrace Dudka Dymond dynbuf @@ -823,12 +824,14 @@ subdirectory submitters substring substrings +sudo SunOS SunSSH superset svc svcb SVCB +SVG Svyatoslav Swisscom sws diff --git a/docs/Makefile.am b/docs/Makefile.am index 862e36536b..554657e889 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -65,6 +65,7 @@ INTERNALDOCS = \ internals/NEW-PROTOCOL.md \ internals/PORTING.md \ internals/README.md \ + internals/SCORECARD.md \ internals/SPLAY.md \ internals/STRPARSE.md \ internals/TLS-SESSIONS.md \ diff --git a/docs/internals/SCORECARD.md b/docs/internals/SCORECARD.md new file mode 100644 index 0000000000..a28b42f040 --- /dev/null +++ b/docs/internals/SCORECARD.md @@ -0,0 +1,71 @@ + + +# scorecard.py + +This is an internal script in `tests/http/scorecard.py` used for testing +curl's performance in a set of cases. These are for exercising parts of +curl/libcurl in a reproducible fashion to judge improvements or detect +regressions. They are not intended to represent real world scenarios +as such. + +This script is not part of any official interface and we may +change it in the future according to the project's needs. + +## setup + +When you are able to run curl's `pytest` suite, scorecard should work +for you as well. They start a local Apache httpd or Caddy server and +invoke the locally build `src/curl` (by default). + +## invocation + +A typical invocation for measuring performance of HTTP/2 downloads would be: + +``` +curl> python3 tests/http/scorecard.py -d h2 +``` + +and this prints a table with the results. The last argument is the protocol to test and +it can be `h1`, `h2` or `h3`. You can add `--json` to get results in JSON instead of text. + +Help for all command line options are available via: + +``` +curl> python3 tests/http/scorecard.py -h +``` + +## scenarios + +Apart from `-d/--downloads` there is `-u/--uploads` and `-r/--requests`. These are run with +a variation of resource sizes and parallelism by default. You can specify these in some way +if you are just interested in a particular case. + +For example, to run downloads of a 1 MB resource only, 100 times with at max 6 parallel transfers, use: + +``` +curl> python3 tests/http/scorecard.py -d --download-sizes=1mb --download-count=100 --download-parallel=6 h2 +``` + +Similar options are available for uploads and requests scenarios. + +## dtrace + +With the `--dtrace` option, scorecard produces a dtrace sample of the user stacks in `tests/http/gen/curl/curl.user_stacks`. On many platforms, `dtrace` requires **special permissions**. It is therefore invoked via `sudo` and you should make sure that sudo works for the run without prompting for a password. + +Note: the file is the trace of the last curl invocation by scorecard. Use the parameters to narrow down the runs to the particular case you are interested in. + +## flame graphs + +With the excellent [Flame Graph](https://github.com/brendangregg/FlameGraph) by Brendan Gregg, scorecard can turn the `dtrace` samples into an interactive SVG. Set the environment variable `FLAMEGRAPH` to the location of your clone of that project and invoked scorecard with the `--flame` option. Like + +``` +curl> FLAMEGRAPH=/Users/sei/projects/FlameGraph python3 tests/http/scorecard.py \ + -r --request-count=50000 --request-parallels=100 --samples=1 --flame h2 +``` +and the SVG of the run is in `tests/http/gen/curl/curl.flamegraph.svg`. You can open that in Firefox and zoom in/out of stacks of interest. + +Note: as with `dtrace`, the flame graph is for the last invocation of curl done by scorecard. diff --git a/tests/http/scorecard.py b/tests/http/scorecard.py index 51a183a162..7f887eda4b 100644 --- a/tests/http/scorecard.py +++ b/tests/http/scorecard.py @@ -186,7 +186,8 @@ class ScoreRunner: curl_verbose: int, download_parallel: int = 0, server_addr: Optional[str] = None, - with_dtrace: bool = False): + with_dtrace: bool = False, + with_flame: bool = False): self.verbose = verbose self.env = env self.protocol = protocol @@ -196,6 +197,7 @@ class ScoreRunner: self._silent_curl = not curl_verbose self._download_parallel = download_parallel self._with_dtrace = with_dtrace + self._with_flame = with_flame def info(self, msg): if self.verbose > 0: @@ -205,7 +207,8 @@ class ScoreRunner: def mk_curl_client(self): return CurlClient(env=self.env, silent=self._silent_curl, server_addr=self.server_addr, - with_dtrace=self._with_dtrace) + with_dtrace=self._with_dtrace, + with_flame=self._with_flame) def handshakes(self) -> Dict[str, Any]: props = {} @@ -679,7 +682,8 @@ def run_score(args, protocol): verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, - with_dtrace=args.dtrace) + with_dtrace=args.dtrace, + with_flame=args.flame) cards.append(card) if test_httpd: @@ -704,7 +708,8 @@ def run_score(args, protocol): server_port=server_port, verbose=args.verbose, curl_verbose=args.curl_verbose, download_parallel=args.download_parallel, - with_dtrace=args.dtrace) + with_dtrace=args.dtrace, + with_flame=args.flame) card.setup_resources(server_docs, downloads) cards.append(card) @@ -808,7 +813,9 @@ def main(): parser.add_argument("--remote", action='store', type=str, default=None, help="score against the remote server at :") parser.add_argument("--dtrace", action='store_true', - default = False, help = "produce dtrace of curl") + default = False, help="produce dtrace of curl") + parser.add_argument("--flame", action='store_true', + default = False, help="produce a flame graph on curl, implies --dtrace") parser.add_argument("-H", "--handshakes", action='store_true', default=False, help="evaluate handshakes only") diff --git a/tests/http/testenv/curl.py b/tests/http/testenv/curl.py index 21bc2fc762..fb28cd7fa7 100644 --- a/tests/http/testenv/curl.py +++ b/tests/http/testenv/curl.py @@ -114,13 +114,17 @@ class DTraceProfile: self._pid = pid self._run_dir = run_dir self._proc = None + self._rc = 0 + self._file = os.path.join(self._run_dir, 'curl.user_stacks') def start(self): + if os.path.exists(self._file): + os.remove(self._file) args = [ 'sudo', 'dtrace', '-x', 'ustackframes=100', '-n', f'profile-97 /pid == {self._pid}/ {{ @[ustack()] = count(); }} tick-60s {{ exit(0); }}', - '-o', f'{self._run_dir}/curl.user_stacks' + '-o', f'{self._file}' ] self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False) assert self._proc @@ -128,6 +132,11 @@ class DTraceProfile: def finish(self): if self._proc: self._proc.terminate() + self._rc = self._proc.returncode + + @property + def file(self): + return self._file class RunTcpDump: @@ -490,7 +499,8 @@ class CurlClient: silent: bool = False, run_env: Optional[Dict[str, str]] = None, server_addr: Optional[str] = None, - with_dtrace: bool = False): + with_dtrace: bool = False, + with_flame: bool = False): self.env = env self._timeout = timeout if timeout else env.test_timeout self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl @@ -500,6 +510,9 @@ class CurlClient: self._headerfile = f'{self._run_dir}/curl.headers' self._log_path = f'{self._run_dir}/curl.log' self._with_dtrace = with_dtrace + self._with_flame = with_flame + if self._with_flame: + self._with_dtrace = True self._silent = silent self._run_env = run_env self._server_addr = server_addr if server_addr else '127.0.0.1' @@ -792,10 +805,11 @@ class CurlClient: exception = None profile = None tcpdump = None - started_at = datetime.now() + dtrace = None if with_tcpdump: tcpdump = RunTcpDump(self.env, self._run_dir) tcpdump.start() + started_at = datetime.now() try: with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr: if with_profile: @@ -824,8 +838,6 @@ class CurlClient: ptimeout = 0.01 exitcode = p.returncode profile.finish() - if self._with_dtrace: - dtrace.finish() log.info(f'done: exit={exitcode}, profile={profile}') else: p = subprocess.run(args, stderr=cerr, stdout=cout, @@ -841,13 +853,18 @@ class CurlClient: f'(configured {self._timeout}s): {args}') exitcode = -1 exception = 'TimeoutExpired' + ended_at = datetime.now() if tcpdump: tcpdump.finish() + if dtrace: + dtrace.finish() + if self._with_flame and dtrace: + self._generate_flame(dtrace, args) coutput = open(self._stdoutfile).readlines() cerrput = open(self._stderrfile).readlines() return ExecResult(args=args, exit_code=exitcode, exception=exception, stdout=coutput, stderr=cerrput, - duration=datetime.now() - started_at, + duration=ended_at - started_at, with_stats=with_stats, profile=profile, tcpdump=tcpdump) @@ -976,3 +993,52 @@ class CurlClient: fin_response(response) return r + + def _generate_flame(self, dtrace: DTraceProfile, curl_args: List[str]): + log.info('generating flame graph from dtrace for this run') + if not os.path.exists(dtrace.file): + raise Exception(f'dtrace output file does not exist: {dtrace.file}') + if 'FLAMEGRAPH' not in os.environ: + raise Exception('Env variable FLAMEGRAPH not set') + fg_dir = os.environ['FLAMEGRAPH'] + if not os.path.exists(fg_dir): + raise Exception(f'FlameGraph directory not found: {fg_dir}') + + fg_collapse = os.path.join(fg_dir, 'stackcollapse.pl') + if not os.path.exists(fg_collapse): + raise Exception(f'FlameGraph script not found: {fg_collapse}') + + fg_gen_flame = os.path.join(fg_dir, 'flamegraph.pl') + if not os.path.exists(fg_gen_flame): + raise Exception(f'FlameGraph script not found: {fg_gen_flame}') + + file_collapsed = f'{dtrace.file}.collapsed' + file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg') + file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr') + log.info('waiting a sec for dtrace to finish flusheing its buffers') + time.sleep(1) + log.info(f'collapsing stacks into {file_collapsed}') + with open(file_collapsed, 'w') as cout, open(file_err, 'w') as cerr: + p = subprocess.run([ + fg_collapse, dtrace.file + ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False) + rc = p.returncode + if rc != 0: + raise Exception(f'{fg_collapse} returned error {rc}') + log.info(f'generating graph into {file_svg}') + cmdline = ' '.join(curl_args) + if len(cmdline) > 80: + title = f'{cmdline[:80]}...' + subtitle = f'...{cmdline[-80:]}' + else: + title = cmdline + subtitle = '' + with open(file_svg, 'w') as cout, open(file_err, 'w') as cerr: + p = subprocess.run([ + fg_gen_flame, '--colors', 'green', + '--title', title, '--subtitle', subtitle, + file_collapsed + ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False) + rc = p.returncode + if rc != 0: + raise Exception(f'{fg_gen_flame} returned error {rc}')