From c149ef5ef9f435a0152e5a8ded581844fc8d5923 Mon Sep 17 00:00:00 2001 From: Lucas Jia Date: Fri, 25 Sep 2026 12:40:25 -0700 Subject: [PATCH 1/2] test(serve): Skip hang-prone endpoint integ tests in PR check The sagemaker-serve-integ-tests PR check hit the 3-hour CodeBuild timeout in 10 of the last 100 builds. In each of those runs, 1 to 3 of the following tests started but never finished, while the other 24 tests completed normally: - test_model_customization_deployment.py::TestModelCustomizationFromModelPackage::test_deploy_from_model_package (9/10) - test_huggingface_integration.py::test_huggingface_build_deploy_invoke_cleanup (6/10) - test_tgi_integration.py::test_tgi_build_deploy_invoke_cleanup (6/10) - test_tei_integration.py::test_tei_build_deploy_invoke_cleanup (6/10) All four deploy real endpoints on GPU instances (ml.g5.xlarge / ml.g6.4xlarge) and are sensitive to instance capacity. Mark them with skip_in_pr_check so they run in the scheduled CI run instead of blocking PR checks. --- sagemaker-serve/tests/integ/test_huggingface_integration.py | 1 + .../tests/integ/test_model_customization_deployment.py | 1 + sagemaker-serve/tests/integ/test_tei_integration.py | 1 + sagemaker-serve/tests/integ/test_tgi_integration.py | 1 + 4 files changed, 4 insertions(+) diff --git a/sagemaker-serve/tests/integ/test_huggingface_integration.py b/sagemaker-serve/tests/integ/test_huggingface_integration.py index acb64d266e..08c3a0785b 100644 --- a/sagemaker-serve/tests/integ/test_huggingface_integration.py +++ b/sagemaker-serve/tests/integ/test_huggingface_integration.py @@ -31,6 +31,7 @@ @pytest.mark.slow_test +@pytest.mark.skip_in_pr_check def test_huggingface_build_deploy_invoke_cleanup(): """Integration test for HuggingFace model build, deploy, invoke, and cleanup workflow""" logger.info("Starting HuggingFace integration test...") diff --git a/sagemaker-serve/tests/integ/test_model_customization_deployment.py b/sagemaker-serve/tests/integ/test_model_customization_deployment.py index 35b1595a57..c388009b49 100644 --- a/sagemaker-serve/tests/integ/test_model_customization_deployment.py +++ b/sagemaker-serve/tests/integ/test_model_customization_deployment.py @@ -325,6 +325,7 @@ def test_build_from_model_package(self, model_package_arn, sagemaker_session): assert model is not None assert model.model_arn is not None + @pytest.mark.skip_in_pr_check def test_deploy_from_model_package( self, model_package_arn, cleanup_endpoints, sagemaker_session ): diff --git a/sagemaker-serve/tests/integ/test_tei_integration.py b/sagemaker-serve/tests/integ/test_tei_integration.py index 9c85b2f02c..9b68325dc9 100644 --- a/sagemaker-serve/tests/integ/test_tei_integration.py +++ b/sagemaker-serve/tests/integ/test_tei_integration.py @@ -31,6 +31,7 @@ @pytest.mark.slow_test +@pytest.mark.skip_in_pr_check def test_tei_build_deploy_invoke_cleanup(): """Integration test for TEI model build, deploy, invoke, and cleanup workflow""" logger.info("Starting TEI integration test...") diff --git a/sagemaker-serve/tests/integ/test_tgi_integration.py b/sagemaker-serve/tests/integ/test_tgi_integration.py index b5774a96c0..fdceef587d 100644 --- a/sagemaker-serve/tests/integ/test_tgi_integration.py +++ b/sagemaker-serve/tests/integ/test_tgi_integration.py @@ -31,6 +31,7 @@ @pytest.mark.slow_test +@pytest.mark.skip_in_pr_check def test_tgi_build_deploy_invoke_cleanup(): """Integration test for TGI model build, deploy, invoke, and cleanup workflow""" logger.info("Starting TGI integration test...") From 7443c269dda0b028aa931727f553435521d3004e Mon Sep 17 00:00:00 2001 From: Lucas Jia Date: Fri, 25 Sep 2026 13:37:55 -0700 Subject: [PATCH 2/2] test(serve): Replace skip_in_pr_check with slow_test marker Consolidate the two markers into one. The old slow_test marker was applied to 19 tests but was never registered in pyproject.toml (the config pytest actually reads) and was not referenced by any CI selection, so it had no effect. skip_in_pr_check carried the real meaning: exclude long-running or hang-prone tests from the PR check. - Remove slow_test from the 19 tests that had it. - Rename skip_in_pr_check to slow_test on the 7 tests that had it. - Register slow_test in pyproject.toml and tox.ini with the former skip_in_pr_check description, and drop the skip_in_pr_check registration. The sagemaker-serve PR-check CodeBuild filter must switch from "not skip_in_pr_check" to "not slow_test" together with this change. --- sagemaker-serve/pyproject.toml | 2 +- .../test_ai_inference_recommender_enhancements_integration.py | 1 - .../tests/integ/test_ai_inference_recommender_integration.py | 2 -- .../test_ai_inference_recommender_sdkt_ic_integration.py | 1 - sagemaker-serve/tests/integ/test_huggingface_integration.py | 1 - sagemaker-serve/tests/integ/test_in_process_integration.py | 1 - sagemaker-serve/tests/integ/test_jumpstart_deploy_parity.py | 2 -- sagemaker-serve/tests/integ/test_jumpstart_integration.py | 1 - .../tests/integ/test_jumpstart_vllm_integration.py | 1 - .../tests/integ/test_model_customization_deployment.py | 4 ++-- sagemaker-serve/tests/integ/test_optimize_integration.py | 2 +- .../integ/test_passthrough_source_code_repack_integration.py | 1 - .../tests/integ/test_private_hub_artifact_resolution.py | 4 ---- sagemaker-serve/tests/integ/test_tei_integration.py | 1 - sagemaker-serve/tests/integ/test_tgi_integration.py | 1 - .../tests/integ/test_train_inference_e2e_integration.py | 1 - sagemaker-serve/tests/integ/test_triton_integration.py | 1 - sagemaker-serve/tox.ini | 3 +-- 18 files changed, 5 insertions(+), 25 deletions(-) diff --git a/sagemaker-serve/pyproject.toml b/sagemaker-serve/pyproject.toml index 9e923f1e0a..58c3c5b97e 100644 --- a/sagemaker-serve/pyproject.toml +++ b/sagemaker-serve/pyproject.toml @@ -74,7 +74,7 @@ python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-v --tb=short" markers = [ - "skip_in_pr_check: mark a test that is excluded from PR check runs. Long-running or hang-prone tests that would otherwise push the run past the CodeBuild timeout; they run in a dedicated scheduled CI run instead.", + "slow_test: mark a test that is excluded from PR check runs. Long-running or hang-prone tests that would otherwise push the run past the CodeBuild timeout; they run in a dedicated scheduled CI run instead.", ] [tool.black] diff --git a/sagemaker-serve/tests/integ/test_ai_inference_recommender_enhancements_integration.py b/sagemaker-serve/tests/integ/test_ai_inference_recommender_enhancements_integration.py index f1360bf25c..d8215c811c 100644 --- a/sagemaker-serve/tests/integ/test_ai_inference_recommender_enhancements_integration.py +++ b/sagemaker-serve/tests/integ/test_ai_inference_recommender_enhancements_integration.py @@ -95,7 +95,6 @@ def test_list_benchmarks_and_recommendations_plumbing(): logger.info("Non-matching filters correctly returned empty lists.") -@pytest.mark.slow_test @pytest.mark.gpu_intensive def test_recommendation_deploy_best_and_compare_e2e(): """Full flow across all three enhancements, sharing one rec job + endpoint: diff --git a/sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py b/sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py index 5a1e4ea8d0..2e4d5f513c 100644 --- a/sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py +++ b/sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py @@ -71,7 +71,6 @@ def _build_jumpstart_model_builder(role_arn): @pytest.mark.slow_test -@pytest.mark.skip_in_pr_check def test_benchmark_workflow_end_to_end(): """Deploy a JumpStart endpoint, run a benchmark against it, parse the result.""" logger.info("Starting AI inference recommender benchmark integration test...") @@ -127,7 +126,6 @@ def test_benchmark_workflow_end_to_end(): ) -@pytest.mark.slow_test @pytest.mark.gpu_intensive def test_recommendation_workflow_end_to_end(): """Run an AI recommendation via generate_deployment_recommendations and deploy the top recommendation.""" diff --git a/sagemaker-serve/tests/integ/test_ai_inference_recommender_sdkt_ic_integration.py b/sagemaker-serve/tests/integ/test_ai_inference_recommender_sdkt_ic_integration.py index 570340b0a4..1582c718d4 100644 --- a/sagemaker-serve/tests/integ/test_ai_inference_recommender_sdkt_ic_integration.py +++ b/sagemaker-serve/tests/integ/test_ai_inference_recommender_sdkt_ic_integration.py @@ -63,7 +63,6 @@ def _src(uri): ] -@pytest.mark.slow_test @pytest.mark.gpu_intensive def test_deploy_sdkt_model_as_inference_component(): """A model carrying SD/KT AdditionalModelDataSources deploys as an diff --git a/sagemaker-serve/tests/integ/test_huggingface_integration.py b/sagemaker-serve/tests/integ/test_huggingface_integration.py index 08c3a0785b..acb64d266e 100644 --- a/sagemaker-serve/tests/integ/test_huggingface_integration.py +++ b/sagemaker-serve/tests/integ/test_huggingface_integration.py @@ -31,7 +31,6 @@ @pytest.mark.slow_test -@pytest.mark.skip_in_pr_check def test_huggingface_build_deploy_invoke_cleanup(): """Integration test for HuggingFace model build, deploy, invoke, and cleanup workflow""" logger.info("Starting HuggingFace integration test...") diff --git a/sagemaker-serve/tests/integ/test_in_process_integration.py b/sagemaker-serve/tests/integ/test_in_process_integration.py index 8f6375821e..d9705f9977 100644 --- a/sagemaker-serve/tests/integ/test_in_process_integration.py +++ b/sagemaker-serve/tests/integ/test_in_process_integration.py @@ -50,7 +50,6 @@ def invoke(self, input_object, model): return {"result": result, "operation": f"multiply by {factor}"} -@pytest.mark.slow_test def test_in_process_build_deploy_invoke_cleanup(): """Integration test for In-Process mode build, deploy, invoke, and cleanup workflow""" logger.info("Starting In-Process integration test...") diff --git a/sagemaker-serve/tests/integ/test_jumpstart_deploy_parity.py b/sagemaker-serve/tests/integ/test_jumpstart_deploy_parity.py index c84a04940b..ab136803b5 100644 --- a/sagemaker-serve/tests/integ/test_jumpstart_deploy_parity.py +++ b/sagemaker-serve/tests/integ/test_jumpstart_deploy_parity.py @@ -26,7 +26,6 @@ MODEL_NAME_PREFIX = "js-netiso-test" -@pytest.mark.slow_test def test_jumpstart_build_enables_network_isolation(): """Integration test verifying JumpStart models are built with EnableNetworkIsolation. @@ -71,7 +70,6 @@ def test_jumpstart_build_enables_network_isolation(): VOLUME_SIZE_INSTANCE_TYPE = "ml.inf2.xlarge" -@pytest.mark.slow_test def test_jumpstart_build_sets_volume_size(): """Integration test verifying volume_size from model specs is propagated. diff --git a/sagemaker-serve/tests/integ/test_jumpstart_integration.py b/sagemaker-serve/tests/integ/test_jumpstart_integration.py index b22c1fedfd..3fdf21e52b 100644 --- a/sagemaker-serve/tests/integ/test_jumpstart_integration.py +++ b/sagemaker-serve/tests/integ/test_jumpstart_integration.py @@ -32,7 +32,6 @@ SERVE_SAGEMAKER_ENDPOINT_TIMEOUT = 15 -@pytest.mark.slow_test @pytest.mark.gpu_intensive def test_jumpstart_build_deploy_invoke_cleanup(): """Integration test for JumpStart model build, deploy, invoke, and cleanup workflow""" diff --git a/sagemaker-serve/tests/integ/test_jumpstart_vllm_integration.py b/sagemaker-serve/tests/integ/test_jumpstart_vllm_integration.py index 861d9326d9..f5e8be8c7e 100644 --- a/sagemaker-serve/tests/integ/test_jumpstart_vllm_integration.py +++ b/sagemaker-serve/tests/integ/test_jumpstart_vllm_integration.py @@ -27,7 +27,6 @@ MODEL_NAME_PREFIX = "js-vllm-test-model" -@pytest.mark.slow_test def test_jumpstart_vllm_build(): """Integration test for JumpStart model using vLLM container image. diff --git a/sagemaker-serve/tests/integ/test_model_customization_deployment.py b/sagemaker-serve/tests/integ/test_model_customization_deployment.py index c388009b49..f8c5b02a2e 100644 --- a/sagemaker-serve/tests/integ/test_model_customization_deployment.py +++ b/sagemaker-serve/tests/integ/test_model_customization_deployment.py @@ -118,7 +118,7 @@ def test_build_from_training_job(self, training_job_name, sagemaker_session): assert model_builder.image_uri is not None assert model_builder.instance_type is not None - @pytest.mark.skip_in_pr_check + @pytest.mark.slow_test def test_deploy_from_training_job(self, training_job_name, sagemaker_session): """Deploy, reuse, invoke, and clean up one training-job endpoint.""" test_id = uuid.uuid4().hex @@ -325,7 +325,7 @@ def test_build_from_model_package(self, model_package_arn, sagemaker_session): assert model is not None assert model.model_arn is not None - @pytest.mark.skip_in_pr_check + @pytest.mark.slow_test def test_deploy_from_model_package( self, model_package_arn, cleanup_endpoints, sagemaker_session ): diff --git a/sagemaker-serve/tests/integ/test_optimize_integration.py b/sagemaker-serve/tests/integ/test_optimize_integration.py index 47b0decea1..bd40616b3e 100644 --- a/sagemaker-serve/tests/integ/test_optimize_integration.py +++ b/sagemaker-serve/tests/integ/test_optimize_integration.py @@ -39,7 +39,7 @@ DJL_LMI_VERSION = "0.31.0" -@pytest.mark.skip_in_pr_check +@pytest.mark.slow_test def test_optimize_build_deploy_invoke_cleanup(): """Integration test for Optimize workflow""" logger.info("Starting Optimize integration test...") diff --git a/sagemaker-serve/tests/integ/test_passthrough_source_code_repack_integration.py b/sagemaker-serve/tests/integ/test_passthrough_source_code_repack_integration.py index 65894a47b5..c5c5826335 100644 --- a/sagemaker-serve/tests/integ/test_passthrough_source_code_repack_integration.py +++ b/sagemaker-serve/tests/integ/test_passthrough_source_code_repack_integration.py @@ -68,7 +68,6 @@ def _tar_members(s3_client, s3_uri): return tarfile.open(fileobj=io.BytesIO(body), mode="r:gz").getnames() -@pytest.mark.slow_test def test_build_repacks_source_code_into_artifact(): """build() with image_uri + model artifact + source_code repacks code/ into the model.tar.gz. No deploy - runs in seconds.""" diff --git a/sagemaker-serve/tests/integ/test_private_hub_artifact_resolution.py b/sagemaker-serve/tests/integ/test_private_hub_artifact_resolution.py index d6c06b7d84..d8133d45de 100644 --- a/sagemaker-serve/tests/integ/test_private_hub_artifact_resolution.py +++ b/sagemaker-serve/tests/integ/test_private_hub_artifact_resolution.py @@ -154,7 +154,6 @@ def execution_role(): return f"arn:aws:iam::{account_id}:role/Admin" -@pytest.mark.slow_test def test_from_jumpstart_config_derives_hub_arn(private_hub, sagemaker_session): """Verify from_jumpstart_config correctly derives hub_arn from hub_name.""" js_config = JumpStartConfig( @@ -176,7 +175,6 @@ def test_from_jumpstart_config_derives_hub_arn(private_hub, sagemaker_session): logger.info("hub_arn correctly derived: %s", mb.hub_arn) -@pytest.mark.slow_test def test_build_resolves_artifacts_via_private_hub(private_hub, execution_role, sagemaker_session): """Verify build() resolves model data through the private hub.""" js_config = JumpStartConfig( @@ -388,7 +386,6 @@ def _deploy_and_assert_hub_access_config( logger.warning("Cleanup failed for %s: %s", kwargs, e) -@pytest.mark.slow_test def test_deploy_with_no_s3_execution_role(private_hub, no_s3_execution_role, sagemaker_session): """E2E: deploy from a private hub with an execution role that has ZERO S3 permissions. Passes only when the SDK attaches HubAccessConfig to @@ -406,7 +403,6 @@ def test_deploy_with_no_s3_execution_role(private_hub, no_s3_execution_role, sag ) -@pytest.mark.slow_test def test_deploy_with_aliased_hub_content_name( private_hub, aliased_model_reference, no_s3_execution_role, sagemaker_session ): diff --git a/sagemaker-serve/tests/integ/test_tei_integration.py b/sagemaker-serve/tests/integ/test_tei_integration.py index 9b68325dc9..9c85b2f02c 100644 --- a/sagemaker-serve/tests/integ/test_tei_integration.py +++ b/sagemaker-serve/tests/integ/test_tei_integration.py @@ -31,7 +31,6 @@ @pytest.mark.slow_test -@pytest.mark.skip_in_pr_check def test_tei_build_deploy_invoke_cleanup(): """Integration test for TEI model build, deploy, invoke, and cleanup workflow""" logger.info("Starting TEI integration test...") diff --git a/sagemaker-serve/tests/integ/test_tgi_integration.py b/sagemaker-serve/tests/integ/test_tgi_integration.py index fdceef587d..b5774a96c0 100644 --- a/sagemaker-serve/tests/integ/test_tgi_integration.py +++ b/sagemaker-serve/tests/integ/test_tgi_integration.py @@ -31,7 +31,6 @@ @pytest.mark.slow_test -@pytest.mark.skip_in_pr_check def test_tgi_build_deploy_invoke_cleanup(): """Integration test for TGI model build, deploy, invoke, and cleanup workflow""" logger.info("Starting TGI integration test...") diff --git a/sagemaker-serve/tests/integ/test_train_inference_e2e_integration.py b/sagemaker-serve/tests/integ/test_train_inference_e2e_integration.py index facc1334fb..ec3eb5a48c 100644 --- a/sagemaker-serve/tests/integ/test_train_inference_e2e_integration.py +++ b/sagemaker-serve/tests/integ/test_train_inference_e2e_integration.py @@ -34,7 +34,6 @@ TRAINING_JOB_PREFIX = "e2e-v3-pytorch" -@pytest.mark.slow_test def test_train_inference_e2e_build_deploy_invoke_cleanup(): """Integration test for Train-Inference E2E workflow""" logger.info("Starting Train-Inference E2E integration test...") diff --git a/sagemaker-serve/tests/integ/test_triton_integration.py b/sagemaker-serve/tests/integ/test_triton_integration.py index b2786254fe..7082231dc6 100644 --- a/sagemaker-serve/tests/integ/test_triton_integration.py +++ b/sagemaker-serve/tests/integ/test_triton_integration.py @@ -44,7 +44,6 @@ def forward(self, x): return torch.softmax(self.linear(x), dim=1) -@pytest.mark.slow_test def test_triton_build_deploy_invoke_cleanup(): """Integration test for Triton model build, deploy, invoke, and cleanup workflow""" logger.info("Starting Triton integration test...") diff --git a/sagemaker-serve/tox.ini b/sagemaker-serve/tox.ini index c5282ddb37..56ea6600f1 100644 --- a/sagemaker-serve/tox.ini +++ b/sagemaker-serve/tox.ini @@ -62,14 +62,13 @@ markers = canary_quick cron local_mode - slow_test + slow_test: mark a test that is excluded from PR check runs. Long-running or hang-prone tests that would otherwise push the run past the CodeBuild timeout; they run in a dedicated scheduled CI run instead. release image_uris_unit_test timeout: mark a test as a timeout. gpu_intensive: mark a test as GPU resource intensive (runs on scheduled CI, not PR checks). us_east_1: mark a test that requires us-east-1 test account credentials (784379639078). import_model: mark a test that creates a Bedrock model import job. Concurrent model import jobs are capped at 1 by a non-raisable Bedrock service quota, so these run serially in a dedicated scheduled CI run, not in PR checks. - skip_in_pr_check: mark a test that is excluded from PR check runs. Long-running or hang-prone tests that would otherwise push the run past the CodeBuild timeout; they run in a dedicated scheduled CI run instead. [testenv] setenv =