Restructure verification targets and type integration test fixtures

Author: kuhe

Makefile

@@ -7,7 +7,60 @@ SMITHY_PYTHON ?= ../smithy-python
 # Clients per gradle build in `make clients`; lower it if the build runs out of heap.
 BATCH_SIZE ?= 25
 
-.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen clients publish-codegen aio-check
+.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen clients publish-codegen aio-check static-analysis test-unit test-functional test-integration
+
+# Each client and package runs in its own .venv. Dependencies outside this repo,
+# including smithy-python, resolve from PyPI at the versions each one declares.
+PROJECTS := $(patsubst %/pyproject.toml,%,$(wildcard clients/*/pyproject.toml packages/*/pyproject.toml))
+RUFF_VERSION := 0.15.22
+PYRIGHT_VERSION := 1.1.406
+UV_RUN = uv run --python $(PYTHON_VERSION) --project
+
+# Q Business is sunsetting; new application creation is unavailable.
+INTEGRATION_SKIP := clients/aws-sdk-qbusiness
+
+static-analysis:
+	@set -e; for p in $(PROJECTS); do \
+		dirs=$$(ls -d $$p/src $$p/tests 2>/dev/null || true); \
+		group=$$(grep -q '^test = \[' $$p/pyproject.toml && echo --group test || true); \
+		echo "== $$p"; \
+		uvx ruff@$(RUFF_VERSION) check --config $$p/pyproject.toml $$dirs; \
+		uvx ruff@$(RUFF_VERSION) format --check --config $$p/pyproject.toml $$dirs; \
+		$(UV_RUN) $$p $$group --with pyright==$(PYRIGHT_VERSION) pyright -p $$p $$dirs; \
+	done
+	# TODO: enable once aws-credentials-sts passes the aio naming rules.
+	# $(MAKE) aio-check
+
+test-unit:
+	@set -e; for p in $(PROJECTS); do \
+		[ -d $$p/tests/unit ] || continue; \
+		echo "== $$p"; \
+		$(UV_RUN) $$p --group test pytest $$p/tests/unit; \
+	done
+
+test-functional:
+	# pytest exits 5 when a generated protocol test file has no cases.
+	@set -e; for p in $(PROJECTS); do \
+		[ -n "$$(find $$p/tests -name 'test_*.py' -not -path '*/tests/unit/*' -not -path '*/tests/integration/*' 2>/dev/null)" ] || continue; \
+		echo "== $$p"; \
+		$(UV_RUN) $$p --group test pytest $$p/tests --ignore=$$p/tests/unit --ignore=$$p/tests/integration || [ $$? -eq 5 ]; \
+	done
+
+# Calls live services, so it needs AWS credentials.
+test-integration:
+	@set -e; for p in $(filter-out $(INTEGRATION_SKIP),$(PROJECTS)); do \
+		[ -d $$p/tests/integration ] || continue; \
+		echo "== $$p"; \
+		$(UV_RUN) $$p --group test --all-extras pytest $$p/tests/integration; \
+	done
+
+# Regenerate every client from codegen/aws-models with codegen built from the
+# smithy-python commit in codegen/smithy-python.toml. Each client is cleared first,
+# keeping only the paths in gen_client.py's KEEP, then scripts/client-overlays is applied.
+clients:
+	SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py --all --pinned --batch-size $(BATCH_SIZE)
+
+########################## Granular targets below ##########################
 
 # Publishes the smithy-python codegen jars to the local Maven cache so `gen`
 # can resolve them. Run once, and again whenever the generator changes.
@@ -19,12 +72,6 @@ publish-codegen:
 gen:
 	SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py $(filter-out gen,$(MAKECMDGOALS))
 
-# Regenerate every client from codegen/aws-models with codegen built from the
-# smithy-python commit in codegen/smithy-python.toml. Each client is cleared first,
-# keeping only the paths in gen_client.py's KEEP, then scripts/client-overlays is applied.
-clients:
-	SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py --all --pinned --batch-size $(BATCH_SIZE)
-
 # Swallow the client-name argument so make does not treat it as a target.
 # Scoped to `gen` so it does not mask typos in unrelated commands.
 ifeq (gen,$(firstword $(MAKECMDGOALS)))

clients/aws-sdk-connecthealth/pyproject.toml

@@ -42,6 +42,7 @@ test = [
     "boto3",
     "pytest>=9.0.1,<10.0.0",
     "pytest-asyncio>=1.3.0,<1.4.0",
+    "types-boto3-lite[s3]",
 ]
 
 [build-system]

clients/aws-sdk-connecthealth/tests/integration/__init__.py

@@ -13,6 +13,9 @@
 REGION = "us-east-1"
 AUDIO_FILE = Path(__file__).parent / "assets" / "test.wav"
 
+# (domain_id, subscription_id, output_s3_uri)
+type ConnectHealthResources = tuple[str, str, str]
+
 
 async def create_connecthealth_client(
     region: str, *, transport: HTTPClient | None = None

clients/aws-sdk-connecthealth/tests/integration/conftest.py

@@ -10,10 +10,12 @@
 
 import asyncio
 import uuid
+from collections.abc import AsyncIterator
 from typing import Any
 
 import boto3
 import pytest
+from types_boto3_s3 import S3Client
 
 from aws_sdk_connecthealth.client import AsyncConnectHealthClient
 from aws_sdk_connecthealth.models import (
@@ -25,7 +27,7 @@
     SubscriptionStatus,
 )
 
-from . import REGION, create_connecthealth_client
+from . import REGION, ConnectHealthResources, create_connecthealth_client
 
 # Tags applied to all resources so orphaned resources from interrupted
 # test runs can be discovered and cleaned up.
@@ -118,7 +120,7 @@ def _create_s3_bucket(s3_client: Any, bucket_name: str) -> None:
     s3_client.create_bucket(
         Bucket=bucket_name,
         CreateBucketConfiguration={
-            "Tags": [{"Key": k, "Value": v} for k, v in _TAGS.items()],
+            "Tags": [{"Key": k, "Value": v} for k, v in _TAGS.items()]
         },
     )
 
@@ -146,13 +148,13 @@ def _delete_s3_bucket(s3_client: Any, bucket_name: str) -> None:
 
 
 @pytest.fixture(scope="session")
-async def connecthealth_resources():
+async def connecthealth_resources() -> AsyncIterator[ConnectHealthResources]:
     """Create ConnectHealth resources for the test session and delete them after."""
     unique_suffix = uuid.uuid4().hex[:16]
     domain_name = f"integ-test-connecthealth-domain-{unique_suffix}"
     bucket_name = f"integ-test-connecthealth-bucket-{unique_suffix}"
 
-    s3_client = boto3.client("s3", region_name=REGION)
+    s3_client: S3Client = boto3.client("s3", region_name=REGION)
     async with await create_connecthealth_client(REGION) as client:
         domain_id: str | None = None
         subscription_id: str | None = None

clients/aws-sdk-connecthealth/tests/integration/test_bidirectional_streaming.py

@@ -39,7 +39,13 @@
     StartMedicalScribeListeningSessionOutput,
 )
 
-from . import AUDIO_FILE, REGION, create_connecthealth_client, streaming_endpoint_plugin
+from . import (
+    AUDIO_FILE,
+    REGION,
+    ConnectHealthResources,
+    create_connecthealth_client,
+    streaming_endpoint_plugin,
+)
 
 SAMPLE_RATE = 16000
 BYTES_PER_SAMPLE = 2
@@ -155,7 +161,9 @@ async def _receive_events(
     return got_transcript
 
 
-async def test_start_medical_scribe_listening_session(connecthealth_resources) -> None:
+async def test_start_medical_scribe_listening_session(
+    connecthealth_resources: ConnectHealthResources,
+) -> None:
     """Test bidirectional streaming with audio input and transcript output."""
     domain_id, subscription_id, output_s3_uri = connecthealth_resources
 

clients/aws-sdk-connecthealth/tests/integration/test_non_streaming.py

@@ -17,10 +17,10 @@
     SubscriptionStatus,
 )
 
-from . import REGION, create_connecthealth_client
+from . import REGION, ConnectHealthResources, create_connecthealth_client
 
 
-async def test_list_domains(connecthealth_resources) -> None:
+async def test_list_domains(connecthealth_resources: ConnectHealthResources) -> None:
     """Test non-streaming ListDomains operation."""
     _ = connecthealth_resources
     async with await create_connecthealth_client(REGION) as client:
@@ -31,7 +31,7 @@ async def test_list_domains(connecthealth_resources) -> None:
         assert len(response.domains) >= 1
 
 
-async def test_get_domain(connecthealth_resources) -> None:
+async def test_get_domain(connecthealth_resources: ConnectHealthResources) -> None:
     """Test non-streaming GetDomain operation."""
     domain_id, _, _ = connecthealth_resources
     async with await create_connecthealth_client(REGION) as client:
@@ -51,7 +51,9 @@ async def test_get_domain(connecthealth_resources) -> None:
         assert response.tags == {"Purpose": "IntegTest"}
 
 
-async def test_list_subscriptions(connecthealth_resources) -> None:
+async def test_list_subscriptions(
+    connecthealth_resources: ConnectHealthResources,
+) -> None:
     """Test non-streaming ListSubscriptions operation."""
     domain_id, subscription_id, _ = connecthealth_resources
     async with await create_connecthealth_client(REGION) as client:
@@ -68,7 +70,9 @@ async def test_list_subscriptions(connecthealth_resources) -> None:
         assert sub.domain_id == domain_id
 
 
-async def test_get_subscription(connecthealth_resources) -> None:
+async def test_get_subscription(
+    connecthealth_resources: ConnectHealthResources,
+) -> None:
     """Test non-streaming GetSubscription operation."""
     domain_id, subscription_id, _ = connecthealth_resources
     async with await create_connecthealth_client(REGION) as client:

clients/aws-sdk-lex-runtime-v2/pyproject.toml

@@ -42,6 +42,7 @@ test = [
     "boto3",
     "pytest>=9.0.1,<10.0.0",
     "pytest-asyncio>=1.3.0,<1.4.0",
+    "types-boto3-lite[iam,lexv2-models,sts]",
 ]
 
 [build-system]

clients/aws-sdk-lex-runtime-v2/tests/integration/conftest.py

@@ -14,6 +14,9 @@
 
 import boto3
 import pytest
+from types_boto3_iam import IAMClient
+from types_boto3_lexv2_models import LexModelsV2Client
+from types_boto3_sts import STSClient
 
 from . import LOCALE_ID, REGION
 
@@ -144,9 +147,9 @@ def lex_bot():
     role_name = f"integ-test-lex-runtime-v2-role-{unique_suffix}"
     bot_name = f"integ-test-lex-runtime-v2-bot-{unique_suffix}"
 
-    iam_client = boto3.client("iam")
-    lex_client = boto3.client("lexv2-models", region_name=REGION)
-    sts_client = boto3.client("sts")
+    iam_client: IAMClient = boto3.client("iam")
+    lex_client: LexModelsV2Client = boto3.client("lexv2-models", region_name=REGION)
+    sts_client: STSClient = boto3.client("sts")
 
     bot_id = None
     try:

clients/aws-sdk-polly/tests/integration/test_bidirectional_streaming.py

@@ -19,7 +19,6 @@
     StartSpeechSynthesisStreamEventStreamServiceQuotaExceededException,
     StartSpeechSynthesisStreamEventStreamStreamClosedEvent,
     StartSpeechSynthesisStreamEventStreamThrottlingException,
-    StartSpeechSynthesisStreamEventStreamUnknown,
     StartSpeechSynthesisStreamEventStreamValidationException,
     StartSpeechSynthesisStreamInput,
     StartSpeechSynthesisStreamOutput,
@@ -85,12 +84,8 @@ async def _receive_audio(
             break
         elif isinstance(event, ERROR_EVENT_TYPES):
             raise event.value
-        elif isinstance(event, StartSpeechSynthesisStreamEventStreamUnknown):
-            raise RuntimeError(f"Received unknown event in stream: {event.tag}")
         else:
-            raise RuntimeError(
-                f"Received unexpected event type in stream: {type(event).__name__}"
-            )
+            raise RuntimeError(f"Received unknown event in stream: {event.tag}")
 
     return audio_bytes, request_characters
 

clients/aws-sdk-qbusiness/tests/integration/conftest.py

@@ -10,9 +10,11 @@
 
 import asyncio
 import uuid
+from collections.abc import AsyncIterator
 
 import pytest
 
+from aws_sdk_qbusiness.client import AsyncQBusinessClient
 from aws_sdk_qbusiness.models import (
     ApplicationStatus,
     CreateApplicationInput,
@@ -46,7 +48,9 @@
 _THROTTLE_RETRY_TIMEOUT_SECONDS = 300
 
 
-async def _wait_for_application_active(client, application_id: str) -> None:
+async def _wait_for_application_active(
+    client: AsyncQBusinessClient, application_id: str
+) -> None:
     """Wait for an Application to reach ACTIVE status.
 
     Args:
@@ -68,7 +72,9 @@ async def _wait_for_application_active(client, application_id: str) -> None:
     raise TimeoutError(f"Application {application_id} did not become ACTIVE in time")
 
 
-async def _wait_for_index_active(client, application_id: str, index_id: str) -> None:
+async def _wait_for_index_active(
+    client: AsyncQBusinessClient, application_id: str, index_id: str
+) -> None:
     """Wait for an Index to reach ACTIVE status.
 
     Args:
@@ -92,7 +98,7 @@ async def _wait_for_index_active(client, application_id: str, index_id: str) ->
 
 
 async def _wait_for_retriever_active(
-    client, application_id: str, retriever_id: str
+    client: AsyncQBusinessClient, application_id: str, retriever_id: str
 ) -> None:
     """Wait for a Retriever to reach ACTIVE status.
 
@@ -119,7 +125,7 @@ async def _wait_for_retriever_active(
 
 
 async def _create_qbusiness_app(
-    client, app_name: str, index_name: str, retriever_name: str
+    client: AsyncQBusinessClient, app_name: str, index_name: str, retriever_name: str
 ) -> str:
     """Create a Q Business application with index and retriever.
 
@@ -185,7 +191,9 @@ async def _create_qbusiness_app(
     return application_id
 
 
-async def _delete_qbusiness_app(client, application_id: str | None) -> None:
+async def _delete_qbusiness_app(
+    client: AsyncQBusinessClient, application_id: str | None
+) -> None:
     """Delete a Q Business application. Cascades to its index and retriever.
 
     Args:
@@ -200,7 +208,7 @@ async def _delete_qbusiness_app(client, application_id: str | None) -> None:
 
 
 @pytest.fixture(scope="session")
-async def qbusiness_app():
+async def qbusiness_app() -> AsyncIterator[str]:
     """Create a Q Business application for the test session and delete it after."""
     unique_suffix = uuid.uuid4().hex[:16]
     app_name = f"integ-test-qbusiness-app-{unique_suffix}"