[3.14] gh-156701: Backport test_dtrace from main (#159596)

* gh-98894: Fix DTrace test_check_probes for shared builds (#151122)

When building with --enable-shared, the SystemTap/DTrace notes
live in libpython. Add detection logic to be used by readelf.

Force the C locale on readelf output.

(cherry picked from commit 3a08e93)

* gh-151496: Use process groups in test_dtrace (#151512)

Create a new process group to run bpftrace commands, so it's possible
to kill also child processes on timeout.

(cherry picked from commit a064b32)

* gh-151496: Use process groups in TraceBackend in test_dtrace (#152039)

Run the generic DTrace/SystemTap commands in a new process group and
terminate the whole group on timeout.

This prevents a forked tracer child from keeping stdout/stderr pipes
open after the direct tracer process is killed.

(cherry picked from commit 1785f4b)

* gh-98894: Check readelf failures in test_dtrace (#152345)

Report readelf command failures directly instead of later failing with
missing-probe assertions.

(cherry picked from commit d60a677)

* gh-98894: Check tracer exit status in test_dtrace (#152893)

Fail functional dtrace and SystemTap cases directly when the tracer exits
non-zero.

(cherry picked from commit 311e7e7)

* gh-98894: Quote test_dtrace tracer subcommands (#152901)

Use shlex.join() when building tracer -c command strings
for avoiding issues with spaces.

(cherry picked from commit a159c68)

* gh-98894: Fix dtrace tests in shared builds (#153372)

Generate SystemTap probe definitions targeting libpython for shared
builds and use centralized USDT probe object discovery for readelf
and BPFTrace.

(cherry picked from commit 298bef3)

* gh-156703: Set LD_LIBRARY_PATH in SystemTap trace commands (GH-156424)

Prefix commands passed to stap -c with LD_LIBRARY_PATH via env, when
probes reside in libpython. This allows the interpreter to find
libpython when staprun does not preserve the loader path.

(cherry picked from commit faf4e52)

* gh-156722, test_dtrace: Terminate tracing processes gracefully on timeout (#156753)

Send SIGTERM to the tracing process group before falling back to SIGKILL.

(cherry picked from commit 55ecc0a)

* gh-156701: Increase the SystemTap usability timeout (#156720)

SystemTap may compile a kernel module during its usability check
(if no cached module is available). We need to allow a larger timeout
for that case.

(cherry picked from commit e1be613)

---------

Co-authored-by: stratakis <cstratak@redhat.com>
Co-authored-by: stratakis <stratakishar@gmail.com>

Author: vstinner

Lib/test/dtracedata/call_stack.stp

@@ -10,7 +10,7 @@ function basename:string(path:string)
     return last_token;
 }
 
-probe process.mark("function__entry")
+probe @PYTHON_SYSTEMTAP_PROBE@("function__entry")
 {
     funcname = user_string($arg2);
 
@@ -19,7 +19,8 @@ probe process.mark("function__entry")
     }
 }
 
-probe process.mark("function__entry"), process.mark("function__return")
+probe @PYTHON_SYSTEMTAP_PROBE@("function__entry"),
+      @PYTHON_SYSTEMTAP_PROBE@("function__return")
 {
     filename = user_string($arg1);
     funcname = user_string($arg2);
@@ -31,7 +32,7 @@ probe process.mark("function__entry"), process.mark("function__return")
     }
 }
 
-probe process.mark("function__return")
+probe @PYTHON_SYSTEMTAP_PROBE@("function__return")
 {
     funcname = user_string($arg2);
 

Lib/test/dtracedata/gc.stp

@@ -1,6 +1,6 @@
 global tracing
 
-probe process.mark("function__entry")
+probe @PYTHON_SYSTEMTAP_PROBE@("function__entry")
 {
     funcname = user_string($arg2);
 
@@ -9,14 +9,15 @@ probe process.mark("function__entry")
     }
 }
 
-probe process.mark("gc__start"), process.mark("gc__done")
+probe @PYTHON_SYSTEMTAP_PROBE@("gc__start"),
+      @PYTHON_SYSTEMTAP_PROBE@("gc__done")
 {
     if (tracing) {
         printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1);
     }
 }
 
-probe process.mark("function__return")
+probe @PYTHON_SYSTEMTAP_PROBE@("function__return")
 {
     funcname = user_string($arg2);
 

Lib/test/test_dtrace.py

@@ -1,14 +1,18 @@
 import dis
 import os.path
 import re
+import shlex
+import signal
 import subprocess
 import sys
 import sysconfig
+import tempfile
 import types
 import unittest
 
 from test import support
 from test.support import findfile, MS_WINDOWS
+from test.support import os_helper
 
 
 if not support.has_subprocess_support:
@@ -23,6 +27,31 @@ def abspath(filename):
     return os.path.abspath(findfile(filename, subdir="dtracedata"))
 
 
+def get_probe_binary():
+    binary = sys.executable
+    if sysconfig.get_config_var("Py_ENABLE_SHARED"):
+        lib_dir = sysconfig.get_config_var("LIBDIR")
+        if not lib_dir or sysconfig.is_python_build():
+            lib_dir = os.path.abspath(os.path.dirname(sys.executable))
+
+        lib_names = []
+        for name in (
+            sysconfig.get_config_var("INSTSONAME"),
+            sysconfig.get_config_var("LDLIBRARY"),
+        ):
+            if name and name not in lib_names:
+                lib_names.append(name)
+
+        if lib_dir:
+            for name in lib_names:
+                libpython_path = os.path.join(lib_dir, name)
+                if os.path.exists(libpython_path):
+                    binary = libpython_path
+                    break
+
+    return binary
+
+
 def normalize_trace_output(output):
     """Normalize DTrace output for comparison.
 
@@ -48,16 +77,77 @@ def normalize_trace_output(output):
         )
 
 
+USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg"))
+TERMINATE_TIMEOUT = 10
+
+def create_process_group(*args, **kwargs):
+    if USE_PROCESS_GROUP:
+        kwargs['start_new_session'] = True
+    return subprocess.Popen(*args, **kwargs)
+
+def terminate_process_group(proc):
+    if USE_PROCESS_GROUP:
+        try:
+            os.killpg(proc.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+    else:
+        proc.terminate()
+
+    try:
+        proc.communicate(timeout=TERMINATE_TIMEOUT)
+    except subprocess.TimeoutExpired:
+        if USE_PROCESS_GROUP:
+            try:
+                os.killpg(proc.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+        else:
+            proc.kill()
+        proc.communicate(timeout=TERMINATE_TIMEOUT)  # Clean up
+
+
+def run_readelf(cmd):
+    # Force the C locale to disable localization.
+    env = dict(os.environ, LC_ALL="C")
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            env=env,
+        )
+    except OSError:
+        raise unittest.SkipTest("Couldn't find readelf on the path")
+
+    with proc:
+        stdout, stderr = proc.communicate()
+
+    if proc.returncode:
+        raise AssertionError(
+            f"Command {shlex.join(cmd)!r} failed "
+            f"with exit code {proc.returncode}: "
+            f"stdout={stdout!r} stderr={stderr!r}"
+        )
+
+    return stdout
+
+
 class TraceBackend:
     EXTENSION = None
     COMMAND = None
     COMMAND_ARGS = []
+    USABILITY_TIMEOUT = 10
 
     def run_case(self, name, optimize_python=None):
-        actual_output = normalize_trace_output(self.trace_python(
-            script_file=abspath(name + self.EXTENSION),
-            python_file=abspath(name + ".py"),
-            optimize_python=optimize_python))
+        try:
+            actual_output = normalize_trace_output(self.trace_python(
+                script_file=abspath(name + self.EXTENSION),
+                python_file=abspath(name + ".py"),
+                optimize_python=optimize_python))
+        except subprocess.TimeoutExpired:
+            raise AssertionError(f"{self.COMMAND[0]} timed out")
 
         with open(abspath(name + self.EXTENSION + ".expected")) as f:
             expected_output = f.read().rstrip()
@@ -70,25 +160,42 @@ def generate_trace_command(self, script_file, subcommand=None):
             command += ["-c", subcommand]
         return command
 
-    def trace(self, script_file, subcommand=None):
+    def trace(self, script_file, subcommand=None, *, timeout=None,
+              check_returncode=False):
         command = self.generate_trace_command(script_file, subcommand)
-        stdout, _ = subprocess.Popen(command,
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.STDOUT,
-                                     universal_newlines=True).communicate()
+        proc = create_process_group(command,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT,
+                                    universal_newlines=True)
+        try:
+            stdout, _ = proc.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            terminate_process_group(proc)
+            raise
+        if check_returncode and proc.returncode:
+            raise AssertionError(
+                f"Command {shlex.join(command)!r} failed "
+                f"with exit code {proc.returncode}: output={stdout!r}"
+            )
         return stdout
 
     def trace_python(self, script_file, python_file, optimize_python=None):
         python_flags = []
         if optimize_python:
             python_flags.extend(["-O"] * optimize_python)
-        subcommand = " ".join([sys.executable] + python_flags + [python_file])
-        return self.trace(script_file, subcommand)
+        subcommand = shlex.join([sys.executable] + python_flags + [python_file])
+        return self.trace(script_file, subcommand, timeout=60,
+                          check_returncode=True)
 
     def assert_usable(self):
         try:
-            output = self.trace(abspath("assert_usable" + self.EXTENSION))
+            output = self.trace(abspath("assert_usable" + self.EXTENSION),
+                                timeout=self.USABILITY_TIMEOUT)
             output = output.strip()
+        except subprocess.TimeoutExpired:
+            raise unittest.SkipTest(
+                f"{self.COMMAND[0]} timed out during usability check"
+            )
         except (FileNotFoundError, NotADirectoryError, PermissionError) as fnfe:
             output = str(fnfe)
         if output != "probe: success":
@@ -105,6 +212,57 @@ class DTraceBackend(TraceBackend):
 class SystemTapBackend(TraceBackend):
     EXTENSION = ".stp"
     COMMAND = ["stap", "-g"]
+    PROBE_PLACEHOLDER = "@PYTHON_SYSTEMTAP_PROBE@"
+    USABILITY_TIMEOUT = 60
+
+    @staticmethod
+    def quote_systemtap_string(value):
+        return value.replace("\\", "\\\\").replace('"', '\\"')
+
+    def python_probe(self):
+        executable = self.quote_systemtap_string(sys.executable)
+        probe_binary = get_probe_binary()
+        if probe_binary == sys.executable:
+            return f'process("{executable}").mark'
+
+        # Python built with --enable-shared
+        probe_binary = self.quote_systemtap_string(probe_binary)
+        return f'process("{executable}").library("{probe_binary}").mark'
+
+    def render_script(self, filename):
+        with open(filename) as fp:
+            script = fp.read()
+
+        return script.replace(self.PROBE_PLACEHOLDER, self.python_probe())
+
+    def generate_trace_command(self, script_file, subcommand=None):
+        probe_binary = get_probe_binary()
+        if subcommand and probe_binary != sys.executable:
+            library_path = os.path.dirname(probe_binary)
+            if existing_path := os.environ.get("LD_LIBRARY_PATH"):
+                library_path = os.pathsep.join((library_path, existing_path))
+            env = shlex.join(["env", f"LD_LIBRARY_PATH={library_path}"])
+            subcommand = f"{env} {subcommand}"
+
+        return super().generate_trace_command(script_file, subcommand)
+
+    def trace(self, script_file, subcommand=None, *, timeout=None,
+              check_returncode=False):
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", suffix=self.EXTENSION, delete=False
+        ) as script:
+            script.write(self.render_script(script_file))
+            generated_script_file = script.name
+
+        try:
+            return super().trace(
+                generated_script_file,
+                subcommand,
+                timeout=timeout,
+                check_returncode=check_returncode,
+            )
+        finally:
+            os_helper.unlink(generated_script_file)
 
 
 @unittest.skipIf(MS_WINDOWS, "Tests not compliant with trace on Windows.")
@@ -189,25 +347,7 @@ def setUpClass(cls):
 
     @staticmethod
     def get_readelf_version():
-        try:
-            cmd = ["readelf", "--version"]
-            proc = subprocess.Popen(
-                cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True,
-            )
-            with proc:
-                version, stderr = proc.communicate()
-
-            if proc.returncode:
-                raise Exception(
-                    f"Command {' '.join(cmd)!r} failed "
-                    f"with exit code {proc.returncode}: "
-                    f"stdout={version!r} stderr={stderr!r}"
-                )
-        except OSError:
-            raise unittest.SkipTest("Couldn't find readelf on the path")
+        version = run_readelf(["readelf", "--version"])
 
         # Regex to parse:
         # 'GNU readelf (GNU Binutils) 2.40.0\n' -> 2.40
@@ -218,14 +358,7 @@ def get_readelf_version():
         return int(match.group(1)), int(match.group(2))
 
     def get_readelf_output(self):
-        command = ["readelf", "-n", sys.executable]
-        stdout, _ = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            universal_newlines=True,
-        ).communicate()
-        return stdout
+        return run_readelf(["readelf", "-n", get_probe_binary()])
 
     def test_check_probes(self):
         readelf_output = self.get_readelf_output()