Provide debug environment and support for retrieving core dumps

.bazelrc

@@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce
 
 test:x86_64-linux --//config:integration_mode=docker
 test:x86_64-linux --//config:unit_mode=host
+# Show a failing test's log (incl. the crash-dump banner) in the console.
+test:x86_64-linux --test_output=errors
 
 # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!)
 build:arm64-linux --config=stub
@@ -181,3 +183,11 @@ test:tsan --test_tag_filters=-no-tsan
 test:tsan --build_tests_only
 test:tsan --cxxopt=-Wno-maybe-uninitialized
 test:tsan --cxxopt=-Wno-redundant-move
+
+# Core-dump capture — opt in with --config=core_dump. The env var gates the
+# plugin at runtime; the build flag selects the gdb-equipped debug image so cores
+# can be analysed in-container and a backtrace is auto-captured. -c dbg is implied
+# so the auto-captured backtrace carries file/line symbols.
+build:core_dump --compilation_mode=dbg
+test:core_dump --test_env=SCORE_ENABLE_CORE_DUMP=1
+test:core_dump --//config:core_dump=True

.gitignore

@@ -71,6 +71,10 @@ target/
 tests/**/*.html
 tests/**/*.xml
 
+# Backup of the kernel core_pattern saved by the integration test core-dump
+# capture (--config=core_dump); auto-removed after restore
+/.original_core_pattern
+
 # IDE Code files
 *.orig
 .venv_docs

MODULE.bazel

@@ -19,6 +19,7 @@ bazel_dep(name = "rules_python", version = "1.8.5")
 bazel_dep(name = "rules_rust", version = "0.68.2-score")
 bazel_dep(name = "rules_cc", version = "0.2.17")
 bazel_dep(name = "rules_oci", version = "2.3.0")
+bazel_dep(name = "rules_distroless", version = "0.8.0")
 bazel_dep(name = "rules_shell", version = "0.6.1")
 bazel_dep(name = "aspect_rules_lint", version = "2.3.0")
 bazel_dep(name = "buildifier_prebuilt", version = "8.5.1")
@@ -175,6 +176,20 @@ oci.pull(
 )
 use_repo(oci, "debian-test-runtime", "debian-test-runtime_linux_amd64")
 
+# gdb (+ dependency closure) layered onto the debug test image; see
+# tests/utils/environments/x86_64-linux/gdb_apt.yaml. Regenerate the lockfile
+# with: bazel run @gdb_apt//:lock
+apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
+apt.install(
+    name = "gdb_apt",
+    lock = "//tests/utils/environments/x86_64-linux:gdb_apt.lock.json",
+    manifest = "//tests/utils/environments/x86_64-linux:gdb_apt.yaml",
+    # Normalize to a merged-usr layout so the layer does not clobber the base
+    # image's /bin, /lib, ... usr-merge symlinks (which would break /bin/sh).
+    mergedusr = True,
+)
+use_repo(apt, "gdb_apt")
+
 bazel_dep(name = "score_baselibs", version = "0.2.10")
 
 # Hedron's Compile Commands Extractor for Bazel

config/BUILD

@@ -50,6 +50,29 @@ config_setting(
     },
 )
 
+# Opt in with --config=core_dump: selects the gdb-equipped debug test image so
+# core dumps can be analysed in-container (see .bazelrc, tests/integration/readme.md).
+bool_flag(
+    name = "core_dump",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "core_dump_enabled",
+    flag_values = {
+        ":core_dump": "True",
+    },
+)
+
+# Specialization of :integration_docker; select() prefers it when both match.
+config_setting(
+    name = "integration_docker_core_dump",
+    flag_values = {
+        ":integration_mode": "docker",
+        ":core_dump": "True",
+    },
+)
+
 # How to run unit tests:
 #
 # - qemu: in a QEMU virtual machine

tests/integration/readme.md

@@ -23,3 +23,120 @@ Currently the following configs are supported:
 - `host`
 - `x86_64-linux`
 
+## Crash dumps (core dumps)
+
+Core-dump capture is **opt-in**: add `--config=core_dump` to include the support. **Attention: This influences the kernel `core_pattern` value of your host system!**
+
+
+How it works:
+- `--config=core_dump` forwards `SCORE_ENABLE_CORE_DUMP=1` into the test
+  environment and sets the `//config:core_dump` build flag (see `.bazelrc`); the
+  shared pytest plugin keys off the env var, individual tests need no adaptions.
+- The build flag selects a **debug image variant** (`score_itf_examples_debug`,
+  the normal image plus `gdb`) so cores can be analysed inside the container.
+  Normal runs keep the slim `score_itf_examples` image, unchanged.
+- The sandbox container runs privileged with an unlimited core-file `ulimit` and
+  a read-write bind-mount of the workspace root.
+- A shared fixture sets the kernel `core_pattern` to a sandbox-local path
+  (`/tmp/score_cores/core.%e.%p.%s.%t`). On teardown it symbolizes each core
+  **inside the container** (where the binary and matching libraries live) into a
+  `<core>.bt.txt` backtrace, copies the cores and backtraces into the Bazel test
+  outputs, and restores the original `core_pattern`.
+- Before changing `core_pattern`, the fixture mirrors the original value to
+  `.original_core_pattern` in the workspace root. The sandboxed test process sees
+  the source tree read-only, so this file is written from inside the privileged
+  container via the workspace bind-mount (hence it is root-owned). It is removed
+  again once the value is restored, so it exists only if a run is force-killed.
+
+Further technical limitations are described in [Important: the `core_pattern` is a global kernel setting](#important-the-core_pattern-is-a-global-kernel-setting)
+
+### Getting a crash dump
+
+Run the (crashing) test with `--config=core_dump`, disabling the cache so it
+actually executes:
+```
+bazel test //tests/integration/<test> --config=x86_64-linux --config=core_dump --nocache_test_results
+```
+
+If a crash dump was created, a `CRASH DUMP` section is printed right under the
+pytest `FAILURES` section at the end of the run (the `x86_64-linux` config
+enables `--test_output=errors`, so the failing log is shown automatically):
+```
+=================================== FAILURES ===================================
+...
+================================== CRASH DUMP ==================================
+CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details.
+
+core.launch_manager.42.6.1787209649:
+    Program terminated with signal SIGABRT, Aborted.
+    #0  0x... in ?? () from /lib/x86_64-linux-gnu/libc.so.6
+    #1  0x... in raise () from /lib/x86_64-linux-gnu/libc.so.6
+    #2  0x... in abort () from /lib/x86_64-linux-gnu/libc.so.6
+    #3  0x... in <your crashing frame> at <file>:<line>
+    ...
+    Full backtrace (all threads): <.../cores/core.launch_manager.*.bt.txt>
+    Reopen in gdb inside the debug image:
+        docker run --rm -it -v <.../launch_manager>:/tmp/.../launch_manager:ro -v <.../cores>:/cores:ro score_itf_examples_debug:latest gdb /tmp/.../launch_manager /cores/core.launch_manager.*
+=========================== short test summary info ============================
+```
+The crashing thread's stack is printed **inline** (symbolized inside the
+container, so libraries match). The printed paths are absolute and
+copy-pasteable. Core files are named `core.<exe>.<pid>.<signal>.<time>` (signal
+`11` = `SIGSEGV`, `6` = `SIGABRT`); they are only produced on an actual crash and
+can be large (hundreds of MB).
+
+### Analysing a crash dump
+
+The crashing thread is already in the banner above. For the **full all-threads**
+dump, open the auto-captured backtrace file the fixture wrote next to the core:
+```
+cat "$(bazel info bazel-testlogs)/tests/integration/<test>/<test>/test.outputs/cores/"*.bt.txt
+```
+`--config=core_dump` builds with `-c dbg` automatically, so the backtrace carries
+file/line information — no extra flag needed.
+
+The banner already prints a ready-to-run `docker run … gdb …` command per core.
+It must run **inside** the debug image (tagged `score_itf_examples_debug:latest`,
+which ships `gdb`), not on the host: the core references the container's
+libraries, so a host `gdb` unwind walks garbage. The command mounts the host
+binary at its original in-container path (so gdb matches the core without
+warnings) and the cores directory, then drops you into an interactive session:
+```
+CORES="$(bazel info bazel-testlogs)/tests/integration/<test>/<test>/test.outputs/cores"
+BIN="$(bazel info bazel-bin)/<path-to-crashing-binary>"
+docker run --rm -it -v "$BIN:/tmp/<remote>/<binary>:ro" -v "$CORES:/cores:ro" \
+    score_itf_examples_debug:latest \
+    gdb /tmp/<remote>/<binary> /cores/core.*
+```
+Running host `gdb` against the core is a last resort; if you must, point it at an
+exported container rootfs via `set sysroot` / `solib-search-path`.
+
+## Important: the `core_pattern` is a global kernel setting
+
+`core_pattern` is **not** namespaced per container - it is shared with the
+host. The fixture therefore changes it globally at the start of a test and
+restores the original value on teardown. Two consequences:
+
+- **Run one crashing target at a time when investigating.** Concurrent tests
+  race on the shared `core_pattern`, so a parallel run can miss dumps or leave
+  a stale value.
+- **If a run is force-killed before teardown**, the restore may not run and
+  `core_pattern` is left pointing at `/tmp/score_cores/...`. The original value
+  was saved to `.original_core_pattern` in the workspace root before it was
+  changed, so you never lose it. **Simply re-running a core-dump test recovers
+  automatically**: the fixture sees the leftover backup, treats it (not the
+  current test value) as the original, and restores it on teardown. To fix it by
+  hand instead:
+  ```
+  cat /proc/sys/kernel/core_pattern          # current (likely /tmp/score_cores/...)
+  cat .original_core_pattern                 # the value to restore
+  # On WSL, write the saved value back from a privileged container:
+  docker run --rm --privileged -v "$PWD/.original_core_pattern:/orig:ro" \
+    debian:bookworm-slim \
+    bash -c 'cat /orig > /proc/sys/kernel/core_pattern'
+  rm .original_core_pattern
+  ```
+  If `.original_core_pattern` is absent, the restore already ran (the file is
+  removed on success). The default `core_pattern` differs per system (e.g.
+  `|/wsl-capture-crash ...` on WSL, `|/lib/systemd/systemd-coredump ...` on
+  systemd hosts, or plain `core`).

tests/utils/bazel/integration.bzl

@@ -75,6 +75,11 @@ def integration_test(
         "//tests/utils/testing_utils",
     ]
     final_data = kwargs.pop("data", []) + [":environment"] + select({
+        # More specialized than :integration_docker; select() prefers it under
+        # --config=core_dump so cores can be analysed against matching libraries.
+        "//config:integration_docker_core_dump": [
+            "//tests/utils/environments/x86_64-linux:x86_64-linux-debug",
+        ],
         "//config:integration_docker": [
             "//tests/utils/environments/x86_64-linux",
         ],
@@ -89,6 +94,10 @@ def integration_test(
         "--score-test-binary-path=$(locations :environment)",
         "--score-test-remote-directory={}/tests/{}".format(install_prefix, name),
     ] + select({
+        "//config:integration_docker_core_dump": [
+            "--docker-image-bootstrap=$(location //tests/utils/environments/x86_64-linux:x86_64-linux-debug)",
+            "--docker-image=score_itf_examples_debug:latest",
+        ],
         "//config:integration_docker": [
             "--docker-image-bootstrap=$(location //tests/utils/environments/x86_64-linux)",
             "--docker-image=score_itf_examples:latest",
@@ -101,11 +110,15 @@ def integration_test(
             "--local-dir=/tmp/score_itf_host/{}".format(name),
         ],
     })
-    final_plugins = ["//tests/utils/plugins:integration_plugin"] + select({
+
+    # integration_plugin is listed last so pytest registers it after the
+    # target plugin: its docker_configuration fixture then overrides the
+    # score_itf default (last-registered -p plugin wins fixture overrides).
+    final_plugins = select({
         "//config:integration_docker": ["@score_itf//score/itf/plugins:docker_plugin"],
         "//config:integration_qemu": ["@score_itf//score/itf/plugins:qemu_plugin"],
         "//config:integration_host": ["//tests/utils/plugins:localhost_plugin"],
-    })
+    }) + ["//tests/utils/plugins:integration_plugin"]
 
     # The QEMU plugin uses a hardcoded port so we can only run one test at a time.
     # See https://github.com/eclipse-score/itf/issues/125.

tests/utils/environments/x86_64-linux/BUILD

@@ -43,3 +43,28 @@ oci_load(
         "//tests:__subpackages__",
     ],
 )
+
+# Debug variant: the normal image plus gdb (and its dependency closure), selected
+# under --config=core_dump so core dumps can be analysed inside the container
+# against matching libraries. The gdb layer is built by rules_distroless from
+# gdb_apt.yaml (regenerate its lockfile with: bazel run @gdb_apt//:lock).
+oci_image(
+    name = "image_debug",
+    base = "@debian-test-runtime",
+    target_compatible_with = ["@platforms//os:linux"],
+    tars = [
+        ":ecu_logging_config_layer",
+        "@gdb_apt//:flat",
+    ],
+)
+
+oci_load(
+    name = "x86_64-linux-debug",
+    image = ":image_debug",
+    repo_tags = ["score_itf_examples_debug:latest"],
+    target_compatible_with = ["@platforms//os:linux"],
+    visibility = [
+        "//examples:__subpackages__",
+        "//tests:__subpackages__",
+    ],
+)