feat(py_test): add opt-in safeguard against accidental no-op tests

.bazelignore

@@ -37,3 +37,4 @@ tests/integration/py_cc_toolchain_registered/bazel-py_cc_toolchain_registered
 tests/integration/toolchain_target_settings/bazel-module_under_test
 tests/integration/unified_pypi/bazel-unified_pypi
 tests/integration/uv_lock/bazel-uv_lock
+tests/integration/validate_test_main/bazel-module_under_test

.bazelrc.deleted_packages

@@ -42,6 +42,7 @@ common --deleted_packages=tests/integration/runtime_manifests
 common --deleted_packages=tests/integration/toolchain_target_settings
 common --deleted_packages=tests/integration/unified_pypi
 common --deleted_packages=tests/integration/uv_lock
+common --deleted_packages=tests/integration/validate_test_main
 common --deleted_packages=tests/modules/another_module
 common --deleted_packages=tests/modules/other
 common --deleted_packages=tests/modules/other/nspkg_delta

docs/api/rules_python/python/config_settings/index.md

@@ -242,6 +242,44 @@ The `auto` value
 The `omit_if_generated_source` value was removed
 ::::
 
+::::{bzl:flag} validate_test_main
+Determines if `py_test` runs a build-time validation that its main module
+actually runs tests.
+
+A common `py_test` pitfall is to define test classes or functions but forget
+to add code that runs them (for example, assuming `py_test` automatically
+invokes `unittest` or `pytest`). When that happens, the test does nothing and
+silently passes.
+
+When enabled, a validation action statically analyzes the main module and
+fails the build if it defines test classes or functions but its top-level body
+is "inert" -- i.e. it only contains definitions, imports, assignments, and
+docstrings, with nothing that actually runs tests (such as an
+`if __name__ == "__main__":` guard that invokes a test runner).
+
+A module that defines no classes or functions at all (for example, one that
+only imports other modules) is always allowed, since it isn't the
+"defined some tests but forgot to run them" case this check targets.
+
+This is only applicable to `py_test` targets that have a `main` source file;
+targets using `main_module` are not checked.
+
+Values:
+
+* `auto`: (default) Automatically decide the effective value; the current
+  behavior is `disabled`.
+* `enabled`: Run the validation action.
+* `disabled`: Don't run the validation action.
+
+:::{note}
+Enabling this requires the exec tools toolchain (with an exec interpreter) to
+be registered, which is the case for the default hermetic toolchains.
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+::::
+
 ::::{bzl:flag} py_linux_libc
 Set what libc is used for the target platform. This will affect which whl binaries will be pulled and what toolchain will be auto-detected. Currently `rules_python` only supplies toolchains compatible with `glibc`.
 

news/3825.added.md

@@ -0,0 +1,6 @@
+(py_test) Added an opt-in safeguard against `py_test` targets that silently
+pass without running any tests. Set
+{obj}`--@rules_python//python/config_settings:validate_test_main=enabled` to
+fail the build when a test's main module only contains inert top-level
+statements (definitions, imports, assignments) and never invokes a test
+runner ([#3824](https://github.com/bazel-contrib/rules_python/issues/3824)).

python/config_settings/BUILD.bazel

@@ -10,6 +10,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "ValidateTestMainFlag",
     "VenvsSitePackages",
     "VenvsUseDeclareSymlinkFlag",
     rp_string_flag = "string_flag",
@@ -77,6 +78,14 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "validate_test_main",
+    build_setting_default = ValidateTestMainFlag.AUTO,
+    values = ValidateTestMainFlag.flag_values(),
+    # NOTE: Only public because it's an implicit dependency of py_test.
+    visibility = ["//visibility:public"],
+)
+
 string_flag(
     name = "precompile_source_retention",
     build_setting_default = PrecompileSourceRetentionFlag.AUTO,

python/private/BUILD.bazel

@@ -18,6 +18,7 @@ load("//python:py_binary.bzl", "py_binary")
 load("//python:py_library.bzl", "py_library")
 load(":bazel_config_mode.bzl", "bazel_config_mode")
 load(":py_exec_tools_toolchain.bzl", "current_interpreter_executable")
+load(":py_interpreter_program.bzl", "py_interpreter_program")
 load(":sentinel_impl.bzl", "sentinel")
 load(":stamp_impl.bzl", "stamp_build_setting")
 load(":uncachable_version_file.bzl", "define_uncachable_version_file")
@@ -219,6 +220,25 @@ py_binary(
     visibility = ["//:__subpackages__"],
 )
 
+# Tool used by py_test's validation action to statically check that the main
+# module actually runs tests. See the validate_test_main config setting.
+py_interpreter_program(
+    name = "py_test_main_validator",
+    main = "py_test_main_validator.py",
+    # Not actually public. Only public because it's an implicit dependency of
+    # the py_test rule.
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "py_test_main_validator_lib",
+    srcs = ["py_test_main_validator.py"],
+    imports = ["../.."],
+    visibility = [
+        "//tests/validate_test_main:__pkg__",
+    ],
+)
+
 bzl_library(
     name = "attr_builders",
     srcs = ["attr_builders.bzl"],

python/private/attributes.bzl

@@ -535,6 +535,14 @@ environment when the test is executed by bazel test.
                 "@platforms//os:watchos",
             ],
         ),
+        "_validate_test_main": lambda: attrb.Label(
+            default = "//python/private:py_test_main_validator",
+            cfg = "exec",
+        ),
+        "_validate_test_main_flag": lambda: attrb.Label(
+            default = labels.VALIDATE_TEST_MAIN,
+            providers = [BuildSettingInfo],
+        ),
     })
 
 # Attributes specific to Python test-equivalent executable rules. Such rules may

python/private/common_labels.bzl

@@ -29,6 +29,7 @@ labels = struct(
     PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")),
     PY_LINUX_LIBC = str(Label("//python/config_settings:py_linux_libc")),
     REPL_DEP = str(Label("//python/bin:repl_dep")),
+    VALIDATE_TEST_MAIN = str(Label("//python/config_settings:validate_test_main")),
     VENV = str(Label("//python/config_settings:venv")),
     VENVS_SITE_PACKAGES = str(Label("//python/config_settings:venvs_site_packages")),
     VENVS_USE_DECLARE_SYMLINK = str(Label("//python/config_settings:venvs_use_declare_symlink")),

python/private/flags.bzl

@@ -87,6 +87,27 @@ AddSrcsToRunfilesFlag = FlagEnum(
     is_enabled = _AddSrcsToRunfilesFlag_is_enabled,
 )
 
+def _ValidateTestMainFlag_is_enabled(ctx):
+    value = ctx.attr._validate_test_main_flag[BuildSettingInfo].value
+    if value == ValidateTestMainFlag.AUTO:
+        # Default off; intended to be flipped to enabled in a future major
+        # version (e.g. rules_python 3.0).
+        value = ValidateTestMainFlag.DISABLED
+    return value == ValidateTestMainFlag.ENABLED
+
+# Determines if py_test runs a validation action that statically checks the
+# main module actually runs tests (instead of silently passing).
+# buildifier: disable=name-conventions
+ValidateTestMainFlag = FlagEnum(
+    # Automatically decide the effective value; currently resolves to disabled.
+    AUTO = "auto",
+    # Run the validation action.
+    ENABLED = "enabled",
+    # Don't run the validation action.
+    DISABLED = "disabled",
+    is_enabled = _ValidateTestMainFlag_is_enabled,
+)
+
 def _string_flag_impl(ctx):
     if ctx.attr.override:
         value = ctx.attr.override

python/private/py_executable.bzl

@@ -57,12 +57,13 @@ load(
     "runfiles_root_path",
 )
 load(":common_labels.bzl", "labels")
-load(":flags.bzl", "BootstrapImplFlag", "VenvsUseDeclareSymlinkFlag", "read_possibly_native_flag")
+load(":flags.bzl", "BootstrapImplFlag", "ValidateTestMainFlag", "VenvsUseDeclareSymlinkFlag", "read_possibly_native_flag")
 load(":precompile.bzl", "maybe_precompile")
 load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo")
 load(":py_executable_info.bzl", "PyExecutableInfo")
 load(":py_info.bzl", "PyInfo", "VenvSymlinkKind")
 load(":py_internal.bzl", "py_internal")
+load(":py_interpreter_program.bzl", "PyInterpreterProgramInfo")
 load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG")
 load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo")
 load(":rule_builders.bzl", "ruleb")
@@ -1170,6 +1171,11 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment =
         main_py = determine_main(ctx)
     else:
         main_py = None
+
+    # Keep a reference to the main source file (before it may be replaced with a
+    # precompiled pyc below) so the test-main validation can statically analyze
+    # the original source.
+    main_py_source = main_py
     direct_sources = filter_to_py_srcs(ctx.files.srcs)
     precompile_result = maybe_precompile(ctx, direct_sources)
 
@@ -1288,10 +1294,84 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment =
         implicit_pyc_source_files = implicit_pyc_source_files,
         imports = imports,
     )
-    _add_provider_output_group_info(providers, py_info, exec_result.output_groups)
+    output_groups = dict(exec_result.output_groups)
+    if is_test:
+        _maybe_add_test_main_validation(ctx, main_py_source, output_groups)
+    _add_provider_output_group_info(providers, py_info, output_groups)
 
     return providers
 
+def _maybe_add_test_main_validation(ctx, main_py, output_groups):
+    """Adds a validation action that checks the test main actually runs tests.
+
+    This is a safeguard against the common pitfall of defining test classes or
+    functions but forgetting to invoke a test runner, which causes the test to
+    silently pass without running anything. See the
+    `//python/config_settings:validate_test_main` flag.
+
+    Args:
+        ctx: Rule ctx.
+        main_py: File or None; the main entry point source file. None when the
+            target uses `main_module` (which can't be statically analyzed here).
+        output_groups: dict[str, depset[File]]; mutated in place to add the
+            `_validation` output group when a validation action is created.
+    """
+    if not ValidateTestMainFlag.is_enabled(ctx):
+        return
+
+    # `main_module` targets execute a module by name; there's no single source
+    # file to statically analyze, so the check doesn't apply.
+    if main_py == None:
+        return
+
+    exec_tools_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE]
+    if exec_tools_toolchain == None or exec_tools_toolchain.exec_tools.exec_interpreter == None:
+        fail(
+            "Validating py_test main modules requires the exec tools toolchain " +
+            "with an exec interpreter, but none was found. Either register one " +
+            "or set --@rules_python//python/config_settings:validate_test_main=disabled.",
+        )
+
+    exec_tools = exec_tools_toolchain.exec_tools
+    validator = ctx.attr._validate_test_main
+    program_info = validator[PyInterpreterProgramInfo]
+    interpreter = exec_tools.exec_interpreter[DefaultInfo].files_to_run
+    validator_files_to_run = validator[DefaultInfo].files_to_run
+
+    validation_output = ctx.actions.declare_file(ctx.label.name + "_validate_test_main.txt")
+
+    args = ctx.actions.args()
+    args.add_all(program_info.interpreter_args)
+    args.add(validator_files_to_run.executable)
+    args.add("--src", main_py)
+    args.add("--src_name", main_py.short_path)
+    args.add("--label", str(ctx.label))
+    args.add("--output", validation_output)
+
+    execution_requirements = {}
+    if testing.ExecutionInfo in validator:
+        execution_requirements = validator[testing.ExecutionInfo].requirements
+
+    ctx.actions.run(
+        executable = interpreter,
+        arguments = [args],
+        inputs = [main_py],
+        outputs = [validation_output],
+        tools = [validator_files_to_run],
+        mnemonic = "PyValidateTestMain",
+        progress_message = "Validating py_test main %{label}",
+        env = program_info.env | {
+            "PYTHONNOUSERSITE": "1",
+            "PYTHONSAFEPATH": "1",
+        },
+        execution_requirements = execution_requirements,
+        toolchain = EXEC_TOOLS_TOOLCHAIN_TYPE,
+    )
+    if "_validation" in output_groups:
+        output_groups["_validation"] = depset([validation_output], transitive = [output_groups["_validation"]])
+    else:
+        output_groups["_validation"] = depset([validation_output])
+
 def _get_build_info(ctx, cc_toolchain):
     build_info_files = py_internal.cc_toolchain_build_info_files(cc_toolchain)
     if cc_helper.is_stamping_enabled(ctx):