scorecard: add perf support on linux

When calling scorecard with --flame to produce a flamegraph, use
"perf" on linux platforms to do the measurements. Update the scorecard
documentation about it.

Closes #19058

diff --git a/docs/internals/SCORECARD.md b/docs/internals/SCORECARD.md
index 7839e0b61f65015e0332158bb9d01164257fb0e7..5469597604179bf5d9605ff9514a5888578a5931 100644 (file)
--- a/docs/internals/SCORECARD.md
+++ b/docs/internals/SCORECARD.md
@@ -59,15 +59,9 @@ If you have configured curl with `--with-test-danted=<danted-path>` for a
 with arguments `--socks4` or `--socks5` to test performance with a SOCKS proxy
 involved. (Note: this does not work for HTTP/3)
 
-## dtrace
-
-With the `--dtrace` option, scorecard produces a dtrace sample of the user stacks in `tests/http/gen/curl/curl.user_stacks`. On many platforms, `dtrace` requires **special permissions**. It is therefore invoked via `sudo` and you should make sure that sudo works for the run without prompting for a password.
-
-Note: the file is the trace of the last curl invocation by scorecard. Use the parameters to narrow down the runs to the particular case you are interested in.
-
 ## flame graphs
 
-With the excellent [Flame Graph](https://github.com/brendangregg/FlameGraph) by Brendan Gregg, scorecard can turn the `dtrace` samples into an interactive SVG. Set the environment variable `FLAMEGRAPH` to the location of your clone of that project and invoked scorecard with the `--flame` option. Like
+With the excellent [Flame Graph](https://github.com/brendangregg/FlameGraph) by Brendan Gregg, scorecard can turn `perf`/`dtrace` samples into an interactive SVG. Either clone the `Flamegraph` repository next to your `curl` project or set the environment variable `FLAMEGRAPH` to the location of your clone. Then run scorecard with the `--flame` option, like
 
 ```
 curl> FLAMEGRAPH=/Users/sei/projects/FlameGraph python3 tests/http/scorecard.py \
@@ -75,4 +69,13 @@ curl> FLAMEGRAPH=/Users/sei/projects/FlameGraph python3 tests/http/scorecard.py
 ```
 and the SVG of the run is in `tests/http/gen/curl/curl.flamegraph.svg`. You can open that in Firefox and zoom in/out of stacks of interest.
 
-Note: as with `dtrace`, the flame graph is for the last invocation of curl done by scorecard.
+The flame graph is about the last run of `curl`. That is why you should add scorecard arguments
+that restrict measurements to a single run.
+
+### Measures/Privileges
+
+The `--flame` option uses `perf` on linux and `dtrace` on macOS. Since both tools require special
+privileges, they are run via the `sudo` command by scorecard. This means you need to issue a
+`sudo` recently enough before running scorecard, so no new password check is needed.
+
+There is no support right now for measurements on other platforms.

diff --git a/tests/http/scorecard.py b/tests/http/scorecard.py
index 9c5417f1bf23c7d4a391c00898c808f4cbfb5847..d987fc586b3f86548e9384752da19d7768c6dfba 100644 (file)
--- a/tests/http/scorecard.py
+++ b/tests/http/scorecard.py
@@ -195,7 +195,6 @@ class ScoreRunner:
                  curl_verbose: int,
                  download_parallel: int = 0,
                  server_addr: Optional[str] = None,
-                 with_dtrace: bool = False,
                  with_flame: bool = False,
                  socks_args: Optional[List[str]] = None,
                  limit_rate: Optional[str] = None):
@@ -207,7 +206,6 @@ class ScoreRunner:
         self.server_port = server_port
         self._silent_curl = not curl_verbose
         self._download_parallel = download_parallel
-        self._with_dtrace = with_dtrace
         self._with_flame = with_flame
         self._socks_args = socks_args
         self._limit_rate = limit_rate
@@ -220,7 +218,6 @@ class ScoreRunner:
     def mk_curl_client(self):
         return CurlClient(env=self.env, silent=self._silent_curl,
                           server_addr=self.server_addr,
-                          with_dtrace=self._with_dtrace,
                           with_flame=self._with_flame,
                           socks_args=self._socks_args)
 
@@ -726,7 +723,6 @@ def run_score(args, protocol):
                                verbose=args.verbose,
                                curl_verbose=args.curl_verbose,
                                download_parallel=args.download_parallel,
-                               with_dtrace=args.dtrace,
                                with_flame=args.flame,
                                socks_args=socks_args,
                                limit_rate=args.limit_rate)
@@ -754,7 +750,6 @@ def run_score(args, protocol):
                                server_port=server_port,
                                verbose=args.verbose, curl_verbose=args.curl_verbose,
                                download_parallel=args.download_parallel,
-                               with_dtrace=args.dtrace,
                                with_flame=args.flame,
                                socks_args=socks_args,
                                limit_rate=args.limit_rate)
@@ -782,7 +777,7 @@ def run_score(args, protocol):
                                server_port=server_port,
                                verbose=args.verbose, curl_verbose=args.curl_verbose,
                                download_parallel=args.download_parallel,
-                               with_dtrace=args.dtrace,
+                               with_flame=args.flame,
                                socks_args=socks_args,
                                limit_rate=args.limit_rate)
             card.setup_resources(server_docs, downloads)
@@ -864,10 +859,8 @@ def main():
                         help="only start the servers")
     parser.add_argument("--remote", action='store', type=str,
                         default=None, help="score against the remote server at <ip>:<port>")
-    parser.add_argument("--dtrace", action='store_true',
-                        default = False, help="produce dtrace of curl")
     parser.add_argument("--flame", action='store_true',
-                        default = False, help="produce a flame graph on curl, implies --dtrace")
+                        default = False, help="produce a flame graph on curl")
     parser.add_argument("--limit-rate", action='store', type=str,
                         default=None, help="use curl's --limit-rate")
 

diff --git a/tests/http/testenv/curl.py b/tests/http/testenv/curl.py
index dcff774a63aea38fc7df460b594d48e2c4983d40..dc885ab8cba92617e77d5956be0e652ef01d0ead 100644 (file)
--- a/tests/http/testenv/curl.py
+++ b/tests/http/testenv/curl.py
@@ -108,6 +108,42 @@ class RunProfile:
                f'stats={self.stats}]'
 
 
+class PerfProfile:
+
+    def __init__(self, pid: int, run_dir):
+        self._pid = pid
+        self._run_dir = run_dir
+        self._proc = None
+        self._rc = 0
+        self._file = os.path.join(self._run_dir, 'curl.perf_stacks')
+
+    def start(self):
+        if os.path.exists(self._file):
+            os.remove(self._file)
+        args = [
+            'sudo', 'perf', 'record', '-F', '99', '-p', f'{self._pid}',
+            '-g', '--', 'sleep', '60'
+        ]
+        self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False)
+        assert self._proc
+
+    def finish(self):
+        if self._proc:
+            self._proc.terminate()
+            self._rc = self._proc.returncode
+        with open(self._file, 'w') as cout:
+            p = subprocess.run([
+                'sudo', 'perf', 'script'
+            ], stdout=cout, cwd=self._run_dir, shell=False)
+            rc = p.returncode
+            if rc != 0:
+                raise Exception(f'perf returned error {rc}')
+
+    @property
+    def file(self):
+        return self._file
+
+
 class DTraceProfile:
 
     def __init__(self, pid: int, run_dir):
@@ -115,7 +151,7 @@ class DTraceProfile:
         self._run_dir = run_dir
         self._proc = None
         self._rc = 0
-        self._file = os.path.join(self._run_dir, 'curl.user_stacks')
+        self._file = os.path.join(self._run_dir, 'curl.dtrace_stacks')
 
     def start(self):
         if os.path.exists(self._file):
@@ -503,6 +539,7 @@ class CurlClient:
                  run_env: Optional[Dict[str, str]] = None,
                  server_addr: Optional[str] = None,
                  with_dtrace: bool = False,
+                 with_perf: bool = False,
                  with_flame: bool = False,
                  socks_args: Optional[List[str]] = None):
         self.env = env
@@ -514,9 +551,21 @@ class CurlClient:
         self._headerfile = f'{self._run_dir}/curl.headers'
         self._log_path = f'{self._run_dir}/curl.log'
         self._with_dtrace = with_dtrace
+        self._with_perf = with_perf
         self._with_flame = with_flame
+        self._fg_dir = None
         if self._with_flame:
-            self._with_dtrace = True
+            self._fg_dir = os.path.join(self.env.project_dir, '../FlameGraph')
+            if 'FLAMEGRAPH' in os.environ:
+                self._fg_dir = os.environ['FLAMEGRAPH']
+            if not os.path.exists(self._fg_dir):
+                raise Exception(f'FlameGraph checkout not found in {self._fg_dir}, set env variable FLAMEGRAPH')
+            if sys.platform.startswith('linux'):
+                self._with_perf = True
+            elif sys.platform.startswith('darwin'):
+                self._with_dtrace = True
+            else:
+                raise Exception(f'flame graphs unsupported on {sys.platform}')
         self._socks_args = socks_args
         self._silent = silent
         self._run_env = run_env
@@ -809,6 +858,7 @@ class CurlClient:
         exception = None
         profile = None
         tcpdump = None
+        perf = None
         dtrace = None
         if with_tcpdump:
             tcpdump = RunTcpDump(self.env, self._run_dir)
@@ -826,7 +876,10 @@ class CurlClient:
                     profile = RunProfile(p.pid, started_at, self._run_dir)
                     if intext is not None and False:
                         p.communicate(input=intext.encode(), timeout=1)
-                    if self._with_dtrace:
+                    if self._with_perf:
+                        perf = PerfProfile(p.pid, self._run_dir)
+                        perf.start()
+                    elif self._with_dtrace:
                         dtrace = DTraceProfile(p.pid, self._run_dir)
                         dtrace.start()
                     ptimeout = 0.0
@@ -860,10 +913,12 @@ class CurlClient:
         ended_at = datetime.now()
         if tcpdump:
             tcpdump.finish()
+        if perf:
+            perf.finish()
         if dtrace:
             dtrace.finish()
-        if self._with_flame and dtrace:
-            self._generate_flame(dtrace, args)
+        if self._with_flame:
+            self._generate_flame(args, dtrace=dtrace, perf=perf)
         coutput = open(self._stdoutfile).readlines()
         cerrput = open(self._stderrfile).readlines()
         return ExecResult(args=args, exit_code=exitcode, exception=exception,
@@ -1004,37 +1059,59 @@ class CurlClient:
         fin_response(response)
         return r
 
-    def _generate_flame(self, dtrace: DTraceProfile, curl_args: List[str]):
-        log.info('generating flame graph from dtrace for this run')
+    def _perf_collapse(self, perf: PerfProfile, file_err):
+        if not os.path.exists(perf.file):
+            raise Exception(f'dtrace output file does not exist: {perf.file}')
+        fg_collapse = os.path.join(self._fg_dir, 'stackcollapse-perf.pl')
+        if not os.path.exists(fg_collapse):
+            raise Exception(f'FlameGraph script not found: {fg_collapse}')
+        stacks_collapsed = f'{perf.file}.collapsed'
+        log.info(f'collapsing stacks into {stacks_collapsed}')
+        with open(stacks_collapsed, 'w') as cout, open(file_err, 'w') as cerr:
+            p = subprocess.run([
+                fg_collapse, perf.file
+            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+            rc = p.returncode
+            if rc != 0:
+                raise Exception(f'{fg_collapse} returned error {rc}')
+        return stacks_collapsed
+
+    def _dtrace_collapse(self, dtrace: DTraceProfile, file_err):
         if not os.path.exists(dtrace.file):
             raise Exception(f'dtrace output file does not exist: {dtrace.file}')
-        if 'FLAMEGRAPH' not in os.environ:
-            raise Exception('Env variable FLAMEGRAPH not set')
-        fg_dir = os.environ['FLAMEGRAPH']
-        if not os.path.exists(fg_dir):
-            raise Exception(f'FlameGraph directory not found: {fg_dir}')
-
-        fg_collapse = os.path.join(fg_dir, 'stackcollapse.pl')
+        fg_collapse = os.path.join(self._fg_dir, 'stackcollapse.pl')
         if not os.path.exists(fg_collapse):
             raise Exception(f'FlameGraph script not found: {fg_collapse}')
-
-        fg_gen_flame = os.path.join(fg_dir, 'flamegraph.pl')
-        if not os.path.exists(fg_gen_flame):
-            raise Exception(f'FlameGraph script not found: {fg_gen_flame}')
-
-        file_collapsed = f'{dtrace.file}.collapsed'
-        file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg')
-        file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr')
-        log.info('waiting a sec for dtrace to finish flusheing its buffers')
-        time.sleep(1)
-        log.info(f'collapsing stacks into {file_collapsed}')
-        with open(file_collapsed, 'w') as cout, open(file_err, 'w') as cerr:
+        stacks_collapsed = f'{dtrace.file}.collapsed'
+        log.info(f'collapsing stacks into {stacks_collapsed}')
+        with open(stacks_collapsed, 'w') as cout, open(file_err, 'a') as cerr:
             p = subprocess.run([
                 fg_collapse, dtrace.file
             ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
             rc = p.returncode
             if rc != 0:
                 raise Exception(f'{fg_collapse} returned error {rc}')
+        return stacks_collapsed
+
+    def _generate_flame(self, curl_args: List[str],
+                        dtrace: Optional[DTraceProfile] = None,
+                        perf: Optional[PerfProfile] = None):
+        fg_gen_flame = os.path.join(self._fg_dir, 'flamegraph.pl')
+        file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg')
+        if not os.path.exists(fg_gen_flame):
+            raise Exception(f'FlameGraph script not found: {fg_gen_flame}')
+
+        log.info('waiting a sec for perf/dtrace to finish flushing')
+        time.sleep(2)
+        log.info('generating flame graph for this run')
+        file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr')
+        if perf:
+            stacks_collapsed = self._perf_collapse(perf, file_err)
+        elif dtrace:
+            stacks_collapsed = self._dtrace_collapse(dtrace, file_err)
+        else:
+            raise Exception('no stacks measure given')
+
         log.info(f'generating graph into {file_svg}')
         cmdline = ' '.join(curl_args)
         if len(cmdline) > 80:
@@ -1043,11 +1120,11 @@ class CurlClient:
         else:
             title = cmdline
             subtitle = ''
-        with open(file_svg, 'w') as cout, open(file_err, 'w') as cerr:
+        with open(file_svg, 'w') as cout, open(file_err, 'a') as cerr:
             p = subprocess.run([
                 fg_gen_flame, '--colors', 'green',
                 '--title', title, '--subtitle', subtitle,
-                file_collapsed
+                stacks_collapsed
             ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
             rc = p.returncode
             if rc != 0: