From 085752d416bfac000bc38c2fc7a9fbf81fd70b3d Mon Sep 17 00:00:00 2001 From: George Fu Date: Fri, 30 Oct 2026 16:53:03 +0000 Subject: [PATCH] Standardize verification and test targets --- .github/workflows/ci.yml | 31 +++--------- CONTRIBUTING.md | 4 +- Makefile | 87 +++++++++++++++++++++------------ codegen/protocol-test/README.md | 2 +- 4 files changed, 66 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea935848c..852f304b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,38 +53,23 @@ jobs: run: | make install - - name: Check python packages + - name: Static analysis if: ${{ matrix.python-version == '3.12' }} run: | - make check-py + make static-analysis - - name: Check API snapshot - if: ${{ matrix.python-version == '3.12' }} + - name: Unit tests run: | - make api-snapshot + make test-unit - - name: Test python packages + - name: Functional tests run: | - make test-py + make test-functional - name: Build python packages run: | make build-py - - name: Check Python codegen output - if: ${{ matrix.python-version == '3.12' }} - run: | - make generate-codegen-output-py - - - name: Build (and test) java packages - run: | - make build-java - - - name: Run protocol tests - run: | - make generate-protocol-tests test-protocols test-integration - - - name: Type-check protocol test stubs - if: ${{ matrix.python-version == '3.12' }} + - name: Integration tests run: | - make typecheck-protocols + make test-integration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1907d16be..71073995c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,8 +31,8 @@ To send us a pull request, please: 1. Fork the repository. 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass (`make test-py`, `make build-java`, and `make test-protocols`). -4. Run `make lint-py` and `make check-py` if you've changed any Python sources. +3. Ensure local tests pass (`make test-unit`, `make test-functional`, and `make test-integration`). +4. Run `make lint-py` and `make static-analysis` if you've changed any Python sources. 5. Run `make format-java` and `make check-java` if you've changed any Java or Gradle sources. 6. Commit to your fork using clear commit messages. 7. Send us a pull request, answering any default questions in the pull request interface. diff --git a/Makefile b/Makefile index 2ee48a516..ff67f75bc 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,49 @@ .DEFAULT_GOAL:=help -.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests generate-codegen-output-py api-snapshot aio-check ci lint-py check-py test-py test-integration benchmark build-py clean +.PHONY: help install format-java check-java build-java generate-protocol-tests generate-codegen-output-py api-snapshot aio-check ci lint-py check-py test-py test-protocols typecheck-protocols static-analysis test-unit test-functional test-integration benchmark build-py clean help: ## Show this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) +# CI runs this target on 3.12 only; see typecheck-protocols. +static-analysis: ## Runs check-py, typecheck-protocols, and api-snapshot. + $(MAKE) check-py + $(MAKE) typecheck-protocols + $(MAKE) api-snapshot + # TODO: enable once aws-sdk-signers passes the aio naming rules. + # $(MAKE) aio-check + + +test-unit: ## Runs the python unit tests. + uv run pytest packages/*/tests/unit + + +# The packages/*/tests/integration suites drive generated clients against +# in-process servers, so they are functional, not live-service, tests. +test-functional: ## Runs the python functional tests and test-protocols. + $(MAKE) test-protocols + uv pip install h2 trustme -e "packages/smithy-http[awscrt]" + uv run pytest packages/*/tests/functional packages/*/tests/integration --no-cov + + +test-integration: ## Builds and tests the Java codegen and asserts the regenerated snapshots match. + $(MAKE) build-java + $(MAKE) generate-protocol-tests + $(MAKE) generate-codegen-output-py + + +ci: ## Runs the full CI sequence locally. + $(MAKE) static-analysis + $(MAKE) test-unit + $(MAKE) test-functional + $(MAKE) build-py + $(MAKE) test-integration + + +########################## Granular targets below ########################## + + install: ## Sets up workspace (* you should run this first! *) uv sync --all-packages --all-extras cd codegen && ./gradlew @@ -24,23 +62,6 @@ build-java: ## Builds and tests the Java code generation packages without modify cd codegen && ./gradlew clean build -test-protocols: ## Generates and runs protocol tests for all supported protocols. - cd codegen && ./gradlew :protocol-test:clean :protocol-test:build - uv pip install -e packages/smithy-test - @set -e; for projection_dir in codegen/protocol-test/build/smithyprojections/protocol-test/*/python-client-codegen; do \ - uv pip install "$$projection_dir"; \ - uv run pytest "$$projection_dir"; \ - done - -# pyright's wrapper imports typing_extensions, which fails to import on 3.15 -# (removed typing.no_type_check_decorator); run the stub type-check on 3.12 only. -typecheck-protocols: ## Type-checks the generated protocol-test stub fixtures with pyright. - @set -e; for projection_dir in codegen/protocol-test/build/smithyprojections/protocol-test/*/python-client-codegen; do \ - uv pip install "$$projection_dir"; \ - uv run pyright "$$projection_dir/tests/_typecheck.py"; \ - done - - generate-protocol-tests: ## Generates protocol-test clients, copies them to ./codegen-output, and asserts no git diff. cd codegen && ./gradlew :protocol-test:clean :protocol-test:build rm -rf codegen-output @@ -79,6 +100,7 @@ generate-codegen-output-py: ## Generates Python output and asserts no snapshot d api-snapshot: ## Records the public API surface of each package and asserts no git diff. uv run python scripts/api-snapshot/api_snapshot.py + aio-check: ## Enforces aio naming and separation in package sources. uv run python scripts/aio-naming/aio_naming.py @@ -94,24 +116,25 @@ check-py: ## Runs checks (formatting, lints, type-checking) on the python packag uv run pyright packages -ci: ## Runs the full CI sequence locally (mirrors the CI workflow steps). - make check-py - make api-snapshot - make test-py - make build-py - make build-java - make generate-codegen-output-py - make generate-protocol-tests test-protocols test-integration - make typecheck-protocols - - test-py: ## Runs tests for the python packages. uv run pytest packages -test-integration: ## Runs integration tests that drive generated clients over a real transport. Requires `make test-protocols` (or generate-protocol-tests) to have installed the generated clients first. - uv pip install h2 trustme "packages/smithy-http[awscrt]" - uv run pytest packages/*/tests/integration --no-cov +test-protocols: ## Runs the generated protocol-test clients in ./codegen-output; run generate-protocol-tests first after codegen changes. + uv pip install -e packages/smithy-test + @set -e; for projection_dir in codegen-output/*/; do \ + uv pip install "$$projection_dir"; \ + uv run pytest "$$projection_dir" --no-cov; \ + done + + +# pyright's wrapper imports typing_extensions, which fails to import on 3.15 +# (removed typing.no_type_check_decorator); run the stub type-check on 3.12 only. +typecheck-protocols: ## Type-checks the generated protocol-test stub fixtures in ./codegen-output with pyright. + @set -e; for projection_dir in codegen-output/*/; do \ + uv pip install "$$projection_dir"; \ + uv run pyright "$$projection_dir/tests/_typecheck.py"; \ + done benchmark: ## Runs ser/deser throughput benchmarks (coverage disabled; not part of CI or the default test run). diff --git a/codegen/protocol-test/README.md b/codegen/protocol-test/README.md index 3ca38ab33..8c178b31f 100644 --- a/codegen/protocol-test/README.md +++ b/codegen/protocol-test/README.md @@ -11,7 +11,7 @@ inputs and outputs. The tests are generated by [`HttpProtocolTestGenerator` For AWS protocols, these tests are defined [here ](https://github.com/awslabs/smithy/tree/main/smithy-aws-protocol-tests). -To run these tests, run `make test-protocols` from the repository root, ensuring that +To run these tests, run `make generate-protocol-tests test-protocols` from the repository root, ensuring that you have a [Python virtual environment](https://docs.python.org/3/library/venv.html) active.