From 019fd2ebbcbc6cd15ee0c341815d4fc5d0fa2bfe Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Mon, 6 Jul 2026 14:29:29 +0200 Subject: [PATCH 1/9] feat: change x86_64 default instance type to c8i.4xlarge Replace c5a.4xlarge (AMD EPYC) with c8i.4xlarge (Intel Xeon 6) as the default x86_64 instance type in DEFAULT_PLATFORM_MAP. The c8i family provides a newer CPU generation with equivalent compute resources (16 vCPUs, 32 GB RAM) and is a prerequisite for enabling nested virtualization support. This change affects KernelCI pull-lab jobs: when the poller translates a job_definition for an x86_64 workload, it will now launch c8i.4xlarge instances instead of c5a.4xlarge. Signed-off-by: Norbert Manthey --- src/kernel_ci_cloud_labs/pull_labs_translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel_ci_cloud_labs/pull_labs_translate.py b/src/kernel_ci_cloud_labs/pull_labs_translate.py index 2695d21..fe78a70 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_translate.py +++ b/src/kernel_ci_cloud_labs/pull_labs_translate.py @@ -24,7 +24,7 @@ # argument of translate_job() to customise per-deployment. DEFAULT_PLATFORM_MAP: Dict[str, Dict[str, str]] = { "x86_64": { - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", }, "arm64": { From 2e52e4f5a04c95ded3332825127ae00baaa4bf16 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Mon, 6 Jul 2026 14:34:10 +0200 Subject: [PATCH 2/9] feat: enable nested virtualization for supported instance families Add automatic detection and enablement of nested virtualization when launching EC2 instances from supported Intel-based families. When a supported instance type is detected, the CpuOptions parameter with NestedVirtualization=enabled is passed to the RunInstances API call. This exposes Intel VT-x to the guest, allowing KVM and Hyper-V workloads to run inside the EC2 instance. Supported families per AWS documentation: C8i, M8i, R8i, X8i, C7i, M7i, R7i, I7i and their variants (flex, d). Reference: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/amazon-ec2-nested-virtualization.html Signed-off-by: Norbert Manthey --- src/kernel_ci_cloud_labs/launch_vm.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/kernel_ci_cloud_labs/launch_vm.py b/src/kernel_ci_cloud_labs/launch_vm.py index ac4add2..8cad9aa 100644 --- a/src/kernel_ci_cloud_labs/launch_vm.py +++ b/src/kernel_ci_cloud_labs/launch_vm.py @@ -175,6 +175,25 @@ def _resolve_ssm_parameter(self, parameter_name): log_error(f"Failed to resolve SSM parameter {parameter_name}: {e}") raise + def _supports_nested_virtualization(self): + """Check if the instance type supports nested virtualization. + + AWS supports nested virtualization on C8i, M8i, R8i, C7i, M7i, R7i + and related instance families. + Returns True if the current instance_type belongs to a supported family. + + See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/amazon-ec2-nested-virtualization.html + """ + nested_families = ( + "c8i", "m8i", "r8i", "c8id", "m8id", "r8id", + "c8i-flex", "m8i-flex", "r8i-flex", "x8i", + "c7i", "m7i", "r7i", "c7id", "m7id", "r7id", + "c7i-flex", "m7i-flex", "i7i", + ) + # Extract family: e.g. "c8i.4xlarge" -> "c8i" + instance_family = self.instance_type.rsplit(".", 1)[0].lower() + return instance_family in nested_families + def prepare_test_artifacts(self): """Verify test payload zip exists in S3.""" log_info(f"\n=== Verifying test artifacts for {self.test} ===") @@ -254,6 +273,12 @@ def spawn_vm(self): params["IamInstanceProfile"] = {"Name": self.role_name} + # Enable nested virtualization for instance families that support it. + # Required for KVM/Hyper-V workloads inside the EC2 instance. + if self._supports_nested_virtualization(): + params["CpuOptions"] = {"NestedVirtualization": "enabled"} + log_info(f"Nested virtualization enabled for {self.instance_type}") + log_not("Calling run_instances...") response = self.ec2.run_instances(**params) From 215c11d30fa695d91e66ad6d221d2a38cc126f84 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Mon, 6 Jul 2026 14:34:35 +0200 Subject: [PATCH 3/9] chore: update example configs to use c8i.4xlarge Update instance_type from c5a.4xlarge to c8i.4xlarge in both example configuration files to match the new default platform map. This ensures local runs and integration tests use the same instance type as KernelCI-triggered pull-lab jobs. Signed-off-by: Norbert Manthey --- examples/aws/alltests-config.json | 16 ++++++++-------- examples/aws/config.json | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/aws/alltests-config.json b/examples/aws/alltests-config.json index 374f9f0..055e70c 100644 --- a/examples/aws/alltests-config.json +++ b/examples/aws/alltests-config.json @@ -205,7 +205,7 @@ "vms": [ { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -215,7 +215,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -225,7 +225,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -235,7 +235,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -245,7 +245,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -255,7 +255,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -265,7 +265,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -275,7 +275,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ diff --git a/examples/aws/config.json b/examples/aws/config.json index 7348f26..3ea2344 100644 --- a/examples/aws/config.json +++ b/examples/aws/config.json @@ -219,7 +219,7 @@ "vms": [ { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ @@ -229,7 +229,7 @@ }, { "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", - "instance_type": "c5a.4xlarge", + "instance_type": "c8i.4xlarge", "root_volume_size": 40, "max_runtime": 3600, "test": [ From 80af59331738158cde0d64196fd125c849ef8c3d Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 7 Jul 2026 14:13:03 +0200 Subject: [PATCH 4/9] fix(ci): update workflows to use 'main' branch instead of 'mainline' The repository default branch is 'main' but all workflows referenced 'mainline', causing them to never trigger on pull requests or pushes. Changes: - basic-ci.yml: trigger on main, drop Python 3.9/3.10 from matrix (package requires >=3.11) - coverage-check.yml: checkout origin/main for base coverage - codeql.yml: trigger on main Signed-off-by: Norbert Manthey --- .github/workflows/basic-ci.yml | 6 +++--- .github/workflows/codeql.yml | 4 ++-- .github/workflows/coverage-check.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/basic-ci.yml b/.github/workflows/basic-ci.yml index 0032d3b..11eea61 100644 --- a/.github/workflows/basic-ci.yml +++ b/.github/workflows/basic-ci.yml @@ -2,9 +2,9 @@ name: Basic CI on: push: - branches: [ mainline ] + branches: [ main ] pull_request: - branches: [ mainline ] + branches: [ main ] permissions: contents: read @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d1fb096..e35a710 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,9 +2,9 @@ name: "CodeQL" on: push: - branches: [ mainline ] + branches: [ main ] pull_request: - branches: [ mainline ] + branches: [ main ] jobs: analyze: diff --git a/.github/workflows/coverage-check.yml b/.github/workflows/coverage-check.yml index a84ea13..87847c5 100644 --- a/.github/workflows/coverage-check.yml +++ b/.github/workflows/coverage-check.yml @@ -29,7 +29,7 @@ jobs: - name: Get base branch coverage run: | - git checkout origin/mainline + git checkout origin/main pip install -e ".[dev]" python -m pytest tests/ -m "not integration" --cov=src --cov-report=json:coverage-base.json --tb=no -q || true BASE_COVERAGE=$(python -c "import json; print(json.load(open('coverage-base.json'))['totals']['percent_covered'])") From e961b783f465b852611e14d5daeaf32ea648d1b0 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 7 Jul 2026 14:24:32 +0200 Subject: [PATCH 5/9] fix: resolve flake8 lint errors - cli.py: remove empty f-string, wrap long line - artifacts.py: remove trailing blank line - pull_labs_poller.py: remove unused submit_tests import - test_kcidb_submit.py: add blank lines before nested definitions - test_pull_labs_poller.py: remove unused tempfile import, fix over-indented continuation lines, rewrite test to not patch the removed submit_tests symbol - test_pull_labs_translate.py: remove unused DEFAULT_TEST_TYPE_MAP import Signed-off-by: Norbert Manthey --- .gitignore | 1 + src/kernel_ci_cloud_labs/cli.py | 7 ++- src/kernel_ci_cloud_labs/core/artifacts.py | 2 - src/kernel_ci_cloud_labs/pull_labs_poller.py | 1 - tests/test_kcidb_submit.py | 2 + tests/test_pull_labs_poller.py | 56 ++++++++------------ tests/test_pull_labs_translate.py | 1 - 7 files changed, 29 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 7ea422a..6c3487e 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,4 @@ analysis/data/ # Distribution archives share/ +test-nested.json diff --git a/src/kernel_ci_cloud_labs/cli.py b/src/kernel_ci_cloud_labs/cli.py index 2187bce..76df4ac 100644 --- a/src/kernel_ci_cloud_labs/cli.py +++ b/src/kernel_ci_cloud_labs/cli.py @@ -292,8 +292,11 @@ def main(): val_parser.add_argument("--bucket", help="S3 bucket to verify (and create with --fix)") val_parser.add_argument("--role", help="IAM role name used by VM instance profiles") val_parser.add_argument("--region", default="us-west-2", help="AWS region (default: us-west-2)") - val_parser.add_argument("--api-url", help=f"KernelCI API base URI (overrides $KERNELCI_API_BASE_URI)") - val_parser.add_argument("--fix", action="store_true", help="Create missing resources (S3 bucket) instead of just reporting them") + val_parser.add_argument("--api-url", help="KernelCI API base URI (overrides $KERNELCI_API_BASE_URI)") + val_parser.add_argument( + "--fix", action="store_true", + help="Create missing resources (S3 bucket) instead of just reporting them", + ) val_parser.set_defaults(func=cmd_setup_validate) args = parser.parse_args() diff --git a/src/kernel_ci_cloud_labs/core/artifacts.py b/src/kernel_ci_cloud_labs/core/artifacts.py index ebe330d..4b4de9b 100644 --- a/src/kernel_ci_cloud_labs/core/artifacts.py +++ b/src/kernel_ci_cloud_labs/core/artifacts.py @@ -262,5 +262,3 @@ def collect_run_artifacts( manifest_path, ) return manifest - - diff --git a/src/kernel_ci_cloud_labs/pull_labs_poller.py b/src/kernel_ci_cloud_labs/pull_labs_poller.py index 124e596..3984d5f 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_poller.py +++ b/src/kernel_ci_cloud_labs/pull_labs_poller.py @@ -45,7 +45,6 @@ from kernel_ci_cloud_labs.kcidb_submit import ( build_test_row, - submit_tests, to_kcidb_status, ) from kernel_ci_cloud_labs.pull_labs_translate import translate_job diff --git a/tests/test_kcidb_submit.py b/tests/test_kcidb_submit.py index 5c2a8d2..5ca0eed 100644 --- a/tests/test_kcidb_submit.py +++ b/tests/test_kcidb_submit.py @@ -187,8 +187,10 @@ def test_submit_revision_posts_with_bearer_auth(self): class FakeResp: def __enter__(self): return self + def __exit__(self, *a): return False + def read(self): return b'{"status":"ok","id":"sub-1"}' diff --git a/tests/test_pull_labs_poller.py b/tests/test_pull_labs_poller.py index 787c341..b5ee0d0 100644 --- a/tests/test_pull_labs_poller.py +++ b/tests/test_pull_labs_poller.py @@ -8,7 +8,6 @@ import json import logging import os -import tempfile import urllib.error from unittest.mock import patch @@ -626,14 +625,16 @@ class TestProcessEventNodeResult: def _run(self, poller, event, translate=None): translate = translate or {"return_value": {}} captured = {} - with patch.object(poller, "_claim_node", return_value=True), \ - patch.object( - poller, "_finish_node", - side_effect=lambda nid, outcome: captured.update(outcome=outcome), - ), \ - patch(_GET, return_value={"artifacts": {}}), \ - patch("kernel_ci_cloud_labs.pull_labs_poller.translate_job", **translate), \ - patch("kernel_ci_cloud_labs.pull_labs_poller.submit_tests", return_value={}): + with ( + patch.object(poller, "_claim_node", return_value=True), + patch.object( + poller, "_finish_node", + side_effect=lambda nid, outcome: captured.update(outcome=outcome), + ), + patch(_GET, return_value={"artifacts": {}}), + patch("kernel_ci_cloud_labs.pull_labs_poller.translate_job", + **translate), + ): poller.process_event(event) return captured["outcome"] @@ -682,9 +683,10 @@ def test_translate_failure_finishes_invalid_job_params(self): assert "missing artifacts.kernel" in outcome.error_msg def test_per_instance_rows_carry_log_url_and_stable_test_id(self): - """When executor returns per-instance rows with log_url, the submitted - KCIDB rows must each carry that URL and a test_id derived from the - instance_id (not the positional index).""" + """When executor returns per-instance rows with log_url, the node + outcome must carry those URLs in artifacts.test_log (first URL) and + test_log_N (subsequent URLs), and the node result must reflect the + aggregated per-instance statuses.""" per_test = [ {"name": "boot", "status": "PASS", "instance_id": "i-aaaa1111", "log_url": "https://b.s3.eu-west-1.amazonaws.com/a.log"}, @@ -695,31 +697,15 @@ def test_per_instance_rows_carry_log_url_and_stable_test_id(self): _minimal_kc(), job_executor=lambda cfg: (per_test, None), ) - seen = {} - with patch.object(p, "_claim_node", return_value=True), \ - patch.object(p, "_finish_node"), \ - patch(_GET, return_value={"artifacts": {}}), \ - patch("kernel_ci_cloud_labs.pull_labs_poller.translate_job", - return_value={}), \ - patch( - "kernel_ci_cloud_labs.pull_labs_poller.submit_tests", - side_effect=lambda url, jwt, origin, build_id, rows: seen.update(rows=rows), - ): - p.process_event(_job_event(node_id="ndX")) - - rows = seen["rows"] - assert len(rows) == 2 - by_id = {r["id"]: r for r in rows} - # test_id derived from instance_id => stable across retries. - assert set(by_id) == {"pullab_cloud_aws:ndX.i-aaaa1111", "pullab_cloud_aws:ndX.i-bbbb2222"} - # Per-row log_url survives the build_test_row pass-through. - assert by_id["pullab_cloud_aws:ndX.i-aaaa1111"]["log_url"] == \ + outcome = self._run(p, _job_event(node_id="ndX")) + + # One FAIL among results -> overall node result is "fail". + assert outcome.result == "fail" + # Log URLs are attached to the outcome artifacts for send_kcidb. + assert outcome.artifacts["test_log"] == \ "https://b.s3.eu-west-1.amazonaws.com/a.log" - assert by_id["pullab_cloud_aws:ndX.i-bbbb2222"]["log_url"] == \ + assert outcome.artifacts["test_log_1"] == \ "https://b.s3.eu-west-1.amazonaws.com/b.log" - # instance_id surfaces in misc for traceability. - assert by_id["pullab_cloud_aws:ndX.i-aaaa1111"]["misc"]["instance_id"] == "i-aaaa1111" - # Aggregated node outcome from per-instance statuses. # (one fail among two -> fail; verified indirectly via existing tests). diff --git a/tests/test_pull_labs_translate.py b/tests/test_pull_labs_translate.py index 68a33a2..b1a6298 100644 --- a/tests/test_pull_labs_translate.py +++ b/tests/test_pull_labs_translate.py @@ -9,7 +9,6 @@ from kernel_ci_cloud_labs.pull_labs_translate import ( DEFAULT_PLATFORM_MAP, - DEFAULT_TEST_TYPE_MAP, translate_job, ) From a30d0d224b94a8a3293973739387287d6d38a915 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 7 Jul 2026 14:33:40 +0200 Subject: [PATCH 6/9] fix: resolve pylint warnings - setup.py: add analysis deps (pandas, matplotlib, seaborn) to [dev] extra so pylint can resolve all imports during linting - pull_labs_poller.py: suppress unused-import on intentional boto3 availability check - setup_validate.py: remove redundant json.JSONDecodeError from except clauses (already covered by parent ValueError) - test_pull_labs_poller.py: disable protected-access check (tests intentionally exercise private methods) - test_role_manager.py: disable protected-access check - test_kcidb_submit.py: suppress unused-argument on mock callbacks that must match urlopen signature Signed-off-by: Norbert Manthey --- setup.py | 3 +++ src/kernel_ci_cloud_labs/pull_labs_poller.py | 2 +- src/kernel_ci_cloud_labs/setup_validate.py | 4 ++-- tests/test_kcidb_submit.py | 4 ++-- tests/test_pull_labs_poller.py | 1 + tests/test_role_manager.py | 1 + 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 8e39de9..8c11b9e 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,9 @@ "isort>=5.0", "pre-commit>=2.0", "pytest-cov>=2.0", + "pandas>=1.3.0", + "matplotlib>=3.4.0", + "seaborn>=0.11.0", ], "analysis": [ "pandas>=1.3.0", diff --git a/src/kernel_ci_cloud_labs/pull_labs_poller.py b/src/kernel_ci_cloud_labs/pull_labs_poller.py index 3984d5f..5057fe6 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_poller.py +++ b/src/kernel_ci_cloud_labs/pull_labs_poller.py @@ -313,7 +313,7 @@ def _validate_default_executor_deps() -> None: problems: List[str] = [] try: - import boto3 # noqa: F401,PLC0415 + import boto3 # noqa: F401,PLC0415 # pylint: disable=unused-import except ImportError as e: problems.append( f"boto3 import failed ({e}) — run: python3.11 -m pip install -e ." diff --git a/src/kernel_ci_cloud_labs/setup_validate.py b/src/kernel_ci_cloud_labs/setup_validate.py index bb51f5b..da0419b 100644 --- a/src/kernel_ci_cloud_labs/setup_validate.py +++ b/src/kernel_ci_cloud_labs/setup_validate.py @@ -200,7 +200,7 @@ def _check_bucket_policy_statement(s3, bucket_name: str, fix: bool) -> bool: print(f"✗ Could not read bucket policy ({code}): {e}") return False existing = None - except (ValueError, json.JSONDecodeError) as e: + except ValueError as e: print(f"✗ Bucket policy is not valid JSON: {e}") return False @@ -365,7 +365,7 @@ def check_kcidb_jwt() -> bool: # JWT payload is base64url; pad to a multiple of 4 before decoding. payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4) payload = json.loads(base64.urlsafe_b64decode(payload_b64)) - except (ValueError, json.JSONDecodeError) as e: + except ValueError as e: print(f"✗ JWT payload not decodable: {e}") return False diff --git a/tests/test_kcidb_submit.py b/tests/test_kcidb_submit.py index 5ca0eed..fd1dfeb 100644 --- a/tests/test_kcidb_submit.py +++ b/tests/test_kcidb_submit.py @@ -239,7 +239,7 @@ def __enter__(self): return self def __exit__(self, *a): return False def read(self): return b'{}' - def fake_urlopen(req, timeout=None): + def fake_urlopen(req, timeout=None): # pylint: disable=unused-argument captured["body"] = json.loads(req.data.decode("utf-8")) return FakeResp() @@ -266,7 +266,7 @@ def __enter__(self): return self def __exit__(self, *a): return False def read(self): return b'{}' - def fake_urlopen(req, timeout=None): + def fake_urlopen(req, timeout=None): # pylint: disable=unused-argument captured["body"] = json.loads(req.data.decode("utf-8")) return FakeResp() diff --git a/tests/test_pull_labs_poller.py b/tests/test_pull_labs_poller.py index b5ee0d0..8b5e980 100644 --- a/tests/test_pull_labs_poller.py +++ b/tests/test_pull_labs_poller.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # # Copyright (C) 2026 Collabora Limited +# pylint: disable=protected-access # Author: Denys Fedoryshchenko """Unit tests for pull_labs_poller (no network, no AWS).""" diff --git a/tests/test_role_manager.py b/tests/test_role_manager.py index c3afbfb..dede843 100644 --- a/tests/test_role_manager.py +++ b/tests/test_role_manager.py @@ -1,4 +1,5 @@ """Unit tests for AWS Role Manager""" +# pylint: disable=protected-access __authors__ = ["Max Hubmann ", "Norbert Manthey "] __copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." From bf3e19d3ebfcb17b5e6acf2754a19c4c95a904c5 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 7 Jul 2026 14:56:40 +0200 Subject: [PATCH 7/9] test: add unit tests for nested virtualization detection Add test_nested_virtualization.py covering _supports_nested_virtualization() in VMLauncher. Tests verify that C8i, M8i, R8i, C7i, M7i, R7i, X8i, I7i families (and flex/d variants) are correctly detected as supporting nested virtualization, while older families (c5a, c6g, t3, m5) are not. Signed-off-by: Norbert Manthey --- tests/test_nested_virtualization.py | 83 +++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/test_nested_virtualization.py diff --git a/tests/test_nested_virtualization.py b/tests/test_nested_virtualization.py new file mode 100644 index 0000000..c664b27 --- /dev/null +++ b/tests/test_nested_virtualization.py @@ -0,0 +1,83 @@ +"""Unit tests for nested virtualization support in launch_vm.""" +# pylint: disable=protected-access + +__authors__ = ["Norbert Manthey "] +__copyright__ = "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved." +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import patch, MagicMock + +import pytest + +from kernel_ci_cloud_labs.launch_vm import VMLauncher + + +@pytest.fixture +def _mock_boto3(): + """Patch boto3 so VMLauncher.__init__ doesn't need real AWS credentials.""" + with patch("kernel_ci_cloud_labs.launch_vm.boto3") as mock_boto: + mock_boto.client.return_value = MagicMock() + yield mock_boto + + +class TestNestedVirtualization: + """Tests for _supports_nested_virtualization and CpuOptions.""" + + def _make_launcher(self, instance_type, _mock_boto3): + """Create a VMLauncher with a given instance_type, mocking AWS.""" + vm_config = { + "instance_type": instance_type, + "ami_id": "ami-test123", + "role_name": "test-role", + "s3_bucket": "test-bucket", + "run_prefix": "run_test", + } + return VMLauncher(vm_config) + + def test_c8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("c8i.4xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_c8i_flex_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("c8i-flex.2xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_m8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("m8i.large", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_r8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("r8i.xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_c7i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("c7i.2xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_m7i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("m7i.xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_x8i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("x8i.large", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_i7i_supports_nested(self, _mock_boto3): + launcher = self._make_launcher("i7i.xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is True + + def test_c5a_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("c5a.4xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is False + + def test_c6g_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("c6g.4xlarge", _mock_boto3) + assert launcher._supports_nested_virtualization() is False + + def test_t3_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("t3.micro", _mock_boto3) + assert launcher._supports_nested_virtualization() is False + + def test_m5_does_not_support_nested(self, _mock_boto3): + launcher = self._make_launcher("m5.large", _mock_boto3) + assert launcher._supports_nested_virtualization() is False From 2d3abf3daa944a6cebef588544d695ec321f71cd Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 7 Jul 2026 15:11:36 +0200 Subject: [PATCH 8/9] fix: add long_description_content_type to setup.py Twine check failed because the content type was missing, defaulting to RST which cannot parse Markdown backticks. Set content type explicitly to text/markdown. Signed-off-by: Norbert Manthey --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8c11b9e..520742b 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ version="1.0.0", description="Kernel-ci-cloud-labs", long_description=long_description, + long_description_content_type="text/markdown", package_dir={"": "src"}, packages=find_packages(where="src"), python_requires=">=3.11", From 37d1e917516c252622542e60a61198cc03506cc2 Mon Sep 17 00:00:00 2001 From: Norbert Manthey Date: Tue, 14 Jul 2026 11:39:37 +0000 Subject: [PATCH 9/9] test: add kvm-unit-tests as a functional virtualization test Run the upstream kvm-unit-tests suite against the guest KVM on a nested-virtualization-enabled instance, as a functional check of the kernel's virtualization stack. The test runs in two stages: run-01 installs the target kernel from the pipeline's shared kernel-rpms area and reboots into it (reusing the flow from unixbench-kernel-regression), and run-02 confirms the kernel switched, then builds QEMU and runs the suite against that kernel's KVM. A hardware gate skips unsupported configurations early, QEMU is built from source because the base image ships no system emulator, and a small stable subset runs by default. Adds the vm-tests/kvm-unit-tests test directory and a c8i.4xlarge VM entry to the example config. The test artifacts are pinned so a daily pipeline run reflects kernel changes only, not test-suite drift: kvm-unit-tests is checked out at a fixed upstream revision (KVMUT_REF, default the commit validated on c8i), and the QEMU source tarball is verified against a pinned sha256 before building. Setting KVMUT_REF empty tracks the upstream default branch instead. Where these artifacts should ultimately come from -- a pinned upstream revision as here, versus a mirrored or vendored copy under our control -- is worth discussing for the merged pipeline. Signed-off-by: Norbert Manthey --- .gitignore | 8 + examples/aws/config.json | 10 + vm-tests/kvm-unit-tests/README.md | 89 ++++ vm-tests/kvm-unit-tests/build-qemu.sh | 97 ++++ vm-tests/kvm-unit-tests/common_lib.sh | 458 ++++++++++++++++++ vm-tests/kvm-unit-tests/dependencies.txt | 7 + .../kvm-unit-tests/external_requirements.json | 4 + .../kvm-unit-tests/run-01-install-kernel.sh | 33 ++ .../kvm-unit-tests/run-02-kvm-unit-tests.sh | 81 ++++ 9 files changed, 787 insertions(+) create mode 100644 vm-tests/kvm-unit-tests/README.md create mode 100755 vm-tests/kvm-unit-tests/build-qemu.sh create mode 100644 vm-tests/kvm-unit-tests/common_lib.sh create mode 100644 vm-tests/kvm-unit-tests/dependencies.txt create mode 100644 vm-tests/kvm-unit-tests/external_requirements.json create mode 100755 vm-tests/kvm-unit-tests/run-01-install-kernel.sh create mode 100755 vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh diff --git a/.gitignore b/.gitignore index 6c3487e..d8be258 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,11 @@ analysis/data/ # Distribution archives share/ test-nested.json + +# kvm-unit-tests run artifacts (SSM logs + fetched sources; not source) +vm-tests/*/ssm-output +vm-tests/*/ssm-output-*/ +vm-tests/kvm-unit-tests/kvm-unit-tests/ +vm-tests/kvm-unit-tests/kernel_version_before.txt +vm-tests/kvm-unit-tests/kvm-unit-tests.log +vm-tests/kvm-unit-tests/results_kvm-unit-tests.* diff --git a/examples/aws/config.json b/examples/aws/config.json index 3ea2344..e23aac7 100644 --- a/examples/aws/config.json +++ b/examples/aws/config.json @@ -236,6 +236,16 @@ "unixbench-kernel-regression" ], "min_count": 1 + }, + { + "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64", + "instance_type": "c8i.4xlarge", + "root_volume_size": 40, + "max_runtime": 5400, + "test": [ + "kvm-unit-tests" + ], + "min_count": 1 } ] } diff --git a/vm-tests/kvm-unit-tests/README.md b/vm-tests/kvm-unit-tests/README.md new file mode 100644 index 0000000..b2fa048 --- /dev/null +++ b/vm-tests/kvm-unit-tests/README.md @@ -0,0 +1,89 @@ +# kvm-unit-tests + +Runs the upstream [kvm-unit-tests](https://gitlab.com/kvm-unit-tests/kvm-unit-tests) +suite against the guest's KVM hypervisor. Intended to run on an EC2 instance +with **nested virtualization enabled** (c8i/c7i and related Intel families, +handled automatically by `_supports_nested_virtualization()` in `launch_vm.py`). + +## What it does + +The test runs in two stages (two `run-*.sh` scripts, with a reboot between them +handled by the pipeline client): + +**run-01 -- install the target kernel.** Records the running kernel, then +installs the kernel under test from the pipeline's shared `kernel-rpms` area +(reusing the flow from `unixbench-kernel-regression`) and makes it the default +boot target; the client then reboots into it. If no kernel RPMs are provided, +the current kernel is kept. + +**run-02 -- run kvm-unit-tests.** After the reboot: + +1. Confirms the running kernel is the one run-01 installed (when applicable). +2. Installs the build toolchain (`git`, `gcc`, `make`, `binutils`) and builds + **QEMU from source** into `/opt/qemu` (`x86_64-softmmu`, `--enable-kvm`) via + `build-qemu.sh`. Amazon Linux 2023 ships no qemu system emulator in its core + repos (only `qemu-img`), and its EPEL9-derived SPAL repo does not carry it + either, so a source build is required. `install_qemu()` exports `QEMU` so + `run_tests.sh` uses the built binary. +3. Verifies `/dev/kvm` is present in the guest -- a built-in regression check on + the EC2 nested-virtualization enablement itself. +4. Clones and builds kvm-unit-tests (pinned to a fixed revision by default via + `KVMUT_REF`; set it empty to track upstream tip). +5. Runs a **curated stable subset** by default (see `KVMUT_TESTS`) with + `ACCEL=kvm ./run_tests.sh -v`. Set `KVMUT_TESTS=""` for the full suite. +6. Writes `results_kvm-unit-tests.txt` (human summary) and + `results_kvm-unit-tests.csv` (`test_name,status,kernel_version,arch`) for + KCIDB ingestion. The parser strips ANSI colour codes from run_tests.sh output + before counting. Both use the `results_` prefix so `test-vm-client.sh` + uploads them; the pass/fail verdict itself is conveyed by the script's exit + code, which the client records as `SUCCESS`/`FAILED` in `result.txt`. + +This is a **functional** test (pass/fail per sub-test), not a benchmark, so it +produces no `benchmark-*.csv` and is not part of the performance-regression +analysis. + +## Why + +kvm-unit-tests is the standard upstream suite for exercising a running kernel's +KVM implementation. Running it continuously in nested-virtualization cloud +guests gives a per-kernel regression signal for the virtualization stack, and +additional test groups can be layered in later via `KVMUT_GROUP`. + +## Validated on c8i (2026-07-10) + +Run on a live `c8i.4xlarge` with nested virt + source-built QEMU 9.2.0: + +- **Full suite:** 54 PASS / 10 FAIL / 23 SKIP. The 10 failures are a coherent + nested-virt-sensitive class: timeouts (`xapic`, `access`, `vmx_apicv_test`, + `vmx_posted_intr_test`, `vmx_pf_exception_test`), MSR/PMU emulation gaps + (`msr`, `msr64`, `pmu`), and nested-VMX instability (`vmx` SIGSEGV, `la57`). +- **Curated subset (default):** 6 PASS / 0 FAIL / 0 SKIP -- clean and stable, + suitable as a hard gate. + +## Two-tier model + +- **Default = stable gate.** `KVMUT_TESTS` runs the proven-stable subset and + `KVMUT_OBSERVE_ONLY=false` gates on it. A regression that breaks one of these + tests fails the pipeline. +- **Full suite = tracking.** Set `KVMUT_TESTS=""`. `KVMUT_IGNORE_FAILURES` is + pre-seeded with the 10 nested-virt-sensitive tests above so a full run still + gates on the stable core; set `KVMUT_OBSERVE_ONLY=true` to only report. + +## Configuration (environment overrides) + +| Variable | Default | Purpose | +|---|---|---| +| `KVMUT_TESTS` | `debug intel_iommu lam vmx_init_signal_test vmx_sipi_signal_test hyperv_clock` | Space-separated test names to run. Empty = full suite. | +| `KVMUT_OBSERVE_ONLY` | `false` | Report counts but never fail. Set `true` for full-suite tracking runs. | +| `KVMUT_IGNORE_FAILURES` | *(10 nested-virt-sensitive tests)* | Test names excluded from the pass/fail decision. | +| `KVMUT_REF` | *(pinned commit `1da1819e`)* | kvm-unit-tests revision for reproducibility. Empty = upstream tip. | +| `KVMUT_GROUP` | *(empty)* | Run only a specific `run_tests.sh -g` group. | +| `QEMU_VERSION` | `9.2.0` | QEMU source version built by `build-qemu.sh`. | + +## Requirements + +- x86_64 instance from a nested-virt-capable family (e.g. `c8i.4xlarge`), + 32 GB+ RAM and ~40 GB disk (QEMU builds in `/tmp`, needs ~3 GB free). +- Kernel RPMs for the kernel under test, supplied by the pipeline + (`external_requirements.json` sets `kernel-rpms/binary: true`). Without them + run-01 keeps the current kernel. diff --git a/vm-tests/kvm-unit-tests/build-qemu.sh b/vm-tests/kvm-unit-tests/build-qemu.sh new file mode 100755 index 0000000..a1cdd4e --- /dev/null +++ b/vm-tests/kvm-unit-tests/build-qemu.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build QEMU from source on AL2023 (no QEMU in default repos). +# Builds a minimal x86_64-softmmu target — enough for nested KVM testing. +set -e +exec 2>&1 + +QEMU_VERSION="${QEMU_VERSION:-9.2.0}" +# Expected sha256 of qemu-${QEMU_VERSION}.tar.xz. download.qemu.org publishes +# only GPG signatures, so the digest is pinned here. This value matches the +# default 9.2.0 tarball; override QEMU_SHA256 when bumping QEMU_VERSION, or set +# it to empty to skip the integrity check. +QEMU_SHA256="${QEMU_SHA256:-f859f0bc65e1f533d040bbe8c92bcfecee5af2c921a6687c652fb44d089bd894}" +QEMU_DIR="/opt/qemu" +NPROC=$(nproc) + +# Pre-flight: disk space check +AVAIL_MB=$(df --output=avail -m /tmp | tail -1 | tr -d ' ') +if [ "$AVAIL_MB" -lt 3072 ]; then + echo "ERROR: Need ≥3GB free in /tmp for QEMU build (have ${AVAIL_MB}MB)" + exit 1 +fi + +if [ -x "$QEMU_DIR/bin/qemu-system-x86_64" ]; then + echo "QEMU already built: $($QEMU_DIR/bin/qemu-system-x86_64 --version | head -1)" + exit 0 +fi + +echo "=== Installing build dependencies ===" +dnf install -y gcc gcc-c++ make ninja-build python3 python3-pip \ + glib2-devel pixman-devel zlib-devel libfdt-devel \ + flex bison diffutils findutils tar gzip xz wget bzip2 +# meson needs a TOML parser on Python <3.11 (AL2023 ships 3.9); tomllib is only +# in the 3.11+ stdlib. Prefer the distro package python3-tomli; fall back to pip +# only if it is not available. +dnf install -y python3-tomli 2>/dev/null || pip3 install tomli 2>/dev/null || true + +# libslirp-devel may not be available — build from source if needed +if ! dnf install -y libslirp-devel 2>/dev/null; then + echo "libslirp-devel not available, building from source..." + dnf install -y meson git 2>/dev/null || pip3 install meson + cd /tmp + wget -q "https://gitlab.freedesktop.org/slirp/libslirp/-/archive/v4.7.0/libslirp-v4.7.0.tar.gz" + tar xf libslirp-v4.7.0.tar.gz + cd libslirp-v4.7.0 + meson setup build --prefix=/usr --default-library=both + ninja -C build install + ldconfig + cd /tmp + rm -rf libslirp-v4.7.0* +fi + +echo "=== Downloading QEMU $QEMU_VERSION ===" +cd /tmp +wget -q "https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz" +if [ -n "$QEMU_SHA256" ]; then + echo "Verifying sha256 of qemu-${QEMU_VERSION}.tar.xz ..." + echo "${QEMU_SHA256} qemu-${QEMU_VERSION}.tar.xz" | sha256sum -c - || { + echo "ERROR: qemu-${QEMU_VERSION}.tar.xz sha256 mismatch (expected ${QEMU_SHA256})" + exit 1 + } +else + echo "WARNING: QEMU_SHA256 is empty -- skipping archive integrity check" +fi +tar xf "qemu-${QEMU_VERSION}.tar.xz" +cd "qemu-${QEMU_VERSION}" + +echo "=== Configuring (x86_64-softmmu only) ===" +./configure \ + --prefix="$QEMU_DIR" \ + --target-list=x86_64-softmmu \ + --enable-kvm \ + --enable-slirp \ + --disable-docs \ + --disable-user \ + --disable-gtk \ + --disable-sdl \ + --disable-opengl \ + --disable-virglrenderer \ + --disable-xen \ + --disable-spice \ + --disable-vnc \ + --disable-curses + +echo "=== Building (${NPROC} jobs) ===" +make -j"$NPROC" +make install + +echo "=== Cleanup ===" +cd / +rm -rf /tmp/qemu-${QEMU_VERSION}* + +echo "QEMU installed: $($QEMU_DIR/bin/qemu-system-x86_64 --version | head -1)" diff --git a/vm-tests/kvm-unit-tests/common_lib.sh b/vm-tests/kvm-unit-tests/common_lib.sh new file mode 100644 index 0000000..226d63f --- /dev/null +++ b/vm-tests/kvm-unit-tests/common_lib.sh @@ -0,0 +1,458 @@ +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Common functions for the kvm-unit-tests test. +# +# This test exercises the KVM hypervisor *inside* an EC2 guest that has nested +# virtualization enabled (c8i/c7i families, see +# _supports_nested_virtualization() in launch_vm.py). The daily KernelCI cloud +# pipeline installs the target kernel, reboots into it, then runs the upstream +# kvm-unit-tests suite against that kernel's KVM -- a per-kernel regression +# signal for the virtualization stack. + +# --------------------------------------------------------------------------- +# Tunables (overridable via environment for experimentation) +# --------------------------------------------------------------------------- +# Upstream source and pinned revision for reproducible runs. +KVMUT_REPO="${KVMUT_REPO:-https://gitlab.com/kvm-unit-tests/kvm-unit-tests.git}" +KVMUT_REPO_MIRROR="${KVMUT_REPO_MIRROR:-https://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git}" +# Pin to a tag/commit for reproducibility so upstream test-suite changes cannot +# break the pipeline. Validated on c8i 2026-07-10 (upstream v2026-04-17-72-g1da1819e). +# Set KVMUT_REF="" to track the default branch tip instead. +KVMUT_REF="${KVMUT_REF:-1da1819e49fc4938985edca67df669099b4c87a7}" +# Directory (persists across reboots in the client work dir). +KVMUT_DIR="${KVMUT_DIR:-${PWD}/kvm-unit-tests}" +# Observe-only mode: when "true", the test reports counts but always exits 0. +# Default is "false" because the default KVMUT_TESTS subset below is a curated, +# proven-stable set (6/6 PASS on c8i 2026-07-10) suitable as a real gate. Set +# "true" when running the FULL suite (KVMUT_TESTS="") to characterise results +# without failing the pipeline. +KVMUT_OBSERVE_ONLY="${KVMUT_OBSERVE_ONLY:-false}" +# Space-separated test names excluded from the pass/fail decision (known flaky +# or nested-virt-sensitive). Defaults to the full-suite failures observed on +# c8i 2026-07-10 so that a full-suite run still gates on the stable core: +# timeouts (xapic, access, vmx_apicv_test, vmx_posted_intr_test, +# vmx_pf_exception_test), MSR/PMU emulation gaps (msr, msr64, pmu), +# nested-VMX instability (vmx SIGSEGV, la57). +KVMUT_IGNORE_FAILURES="${KVMUT_IGNORE_FAILURES-xapic access vmx_apicv_test vmx_posted_intr_test vmx_pf_exception_test msr msr64 pmu vmx la57}" +# Optional explicit test group to run (passed to run_tests.sh -g). Empty = all. +KVMUT_GROUP="${KVMUT_GROUP:-}" +# Explicit subset of test names to run (space-separated, passed positionally to +# run_tests.sh). Defaults to a small, fast, nested-virt-stable smoke set proven +# to pass on c8i (2026-07-10). Set KVMUT_TESTS="" to run the full suite, or +# KVMUT_TESTS="a b c" to pick your own. +KVMUT_TESTS="${KVMUT_TESTS-debug intel_iommu lam vmx_init_signal_test vmx_sipi_signal_test hyperv_clock}" + +# Error trap handler to show the line where an error occurred +error_trap() +{ + local exit_code=$? + local line_number=$1 + echo "$(date): ERROR: Script failed at line $line_number with exit code $exit_code" + echo "$(date): ERROR: Command that failed: $(sed -n "${line_number}p" "$0")" + exit $exit_code +} +trap 'error_trap $LINENO' ERR + +# Install a single given package (yum first, dnf fallback) +install_package() +{ + local pkg="$1" + local output + echo "Installing package $pkg ..." + if output=$(sudo yum install -y "$pkg" 2>&1) || output=$(sudo dnf install -y "$pkg" 2>&1); then + return 0 + else + echo "Failed to install package $pkg:" + echo "$output" + return 1 + fi +} + +# Install all packages listed in dependencies.txt +install_test_dependencies() +{ + local deps_file="${SOURCE_DIR}/dependencies.txt" + if [ -f "$deps_file" ]; then + while IFS= read -r pkg || [ -n "$pkg" ]; do + [[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue + pkg=$(echo "$pkg" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$pkg" ] && { install_package "$pkg" || return 1; } + done <"$deps_file" + fi +} + +# --------------------------------------------------------------------------- +# Kernel management (reused from unixbench-kernel-regression): install the +# target kernel from the pipeline's shared kernel-rpms area and boot into it, +# so the suite runs against the kernel under test rather than the AMI default. +# --------------------------------------------------------------------------- +# Results bucket + kernel paths, populated from the pipeline environment. +RESULTS_BUCKET="${S3_BUCKET:-}" +ARCH="$(uname -m)" +KERNEL_RPM_DIR="/tmp/kernel-rpms" +KERNEL_FILE="${SOURCE_DIR}/kernel_version_before.txt" + +get_running_kernel() +{ + uname -r +} + +save_kernel_version() +{ + local version="$1" + local out_file="$2" + if [ -z "$version" ] || [ -z "$out_file" ]; then + echo "ERROR: save_kernel_version requires version and file" >&2 + return 1 + fi + echo "$version" >"$out_file" +} + +load_kernel_version() +{ + local in_file="$1" + if [ ! -f "$in_file" ]; then + echo "ERROR: Kernel version file not found: $in_file" >&2 + return 1 + fi + cat "$in_file" +} + +assert_kernel_changed() +{ + local before="$1" + local after="$2" + if [ "$before" = "$after" ]; then + echo "ERROR: kernel version did not change (still $after)" >&2 + return 1 + fi + echo "Kernel version changed from $before to $after" +} + +# List available kernel RPMs from the shared S3 area. +list_kernels_from_s3() +{ + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX:-}/shared/kernel-rpms/binary/${ARCH}/" + aws s3 ls "${S3_PATH}" | grep "\.rpm$" | awk '{print $4}' +} + +# Download a specific kernel RPM from S3. +download_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: download_kernel_rpm requires kernel_name parameter" >&2 + return 1 + fi + local kernel_name="$1" + S3_PATH="s3://${RESULTS_BUCKET}/${RUN_PREFIX:-}/shared/kernel-rpms/binary/${ARCH}/" + mkdir -p "$KERNEL_RPM_DIR" + local local_path="${KERNEL_RPM_DIR}/${kernel_name}" + if [ -f "$local_path" ]; then + echo "$local_path" + return 0 + fi + if aws s3 cp "${S3_PATH}${kernel_name}" "$local_path" --no-progress >&2; then + echo "$local_path" + return 0 + else + echo "ERROR: Failed to download kernel" >&2 + return 1 + fi +} + +# Return the lowest-versioned kernel RPM (downloads it from S3). +get_first_kernel_rpm_from_dir() +{ + local kernels=$(list_kernels_from_s3 | sort -V) + local first_kernel=$(echo "$kernels" | head -n 1) + if [ -z "$first_kernel" ]; then + return 1 + fi + download_kernel_rpm "$first_kernel" +} + +# Dump boot configuration for debugging kernel install issues. +dump_boot_info() +{ + echo "=== Boot Debug Info ===" + echo "--- OS ---" + head -2 /etc/os-release 2>/dev/null || true + echo "--- Running kernel ---" + uname -r + echo "--- Installed kernel packages ---" + rpm -qa 'kernel*' | sort + echo "--- vmlinuz files in /boot ---" + ls -la /boot/vmlinuz-* 2>/dev/null || echo "(none)" + echo "--- BLS entries ---" + ls -la /boot/loader/entries/ 2>/dev/null || echo "(no BLS directory)" + echo "--- grubby default ---" + sudo grubby --default-kernel 2>/dev/null || echo "(grubby --default-kernel failed)" + echo "=== End Boot Debug Info ===" +} + +# Install a kernel RPM and make it the default boot target. +install_kernel_rpm() +{ + if [ -z "${1:-}" ]; then + echo "ERROR: install_kernel_rpm requires kernel_rpm parameter" >&2 + return 1 + fi + local kernel_rpm="$1" + + local host_arch=$(uname -m) + local rpm_arch=$(rpm -qp --queryformat '%{ARCH}' "$kernel_rpm" 2>/dev/null) + if [ "$rpm_arch" != "$host_arch" ]; then + echo "ERROR: Architecture mismatch - Host: $host_arch, RPM: $rpm_arch" >&2 + return 1 + fi + + echo "kernel before installation: $(uname -r)" + echo "Installing kernel from $kernel_rpm (arch: $rpm_arch)" + + if sudo yum localinstall -y "$kernel_rpm" 2>/dev/null || sudo dnf install -y "$kernel_rpm" 2>/dev/null; then + dump_boot_info + + # Set the newly installed kernel as default boot target. Without this, + # GRUB may boot a different kernel than the one just installed. + local installed_version + installed_version=$(rpm -qp --queryformat '%{VERSION}' "$kernel_rpm" 2>/dev/null) + + local grub_kernel + grub_kernel=$(sudo grubby --info=ALL 2>/dev/null \ + | grep "^kernel=" \ + | grep "$installed_version" \ + | head -1 \ + | sed 's/^kernel=//' \ + | tr -d '"' \ + || true) + + if [ -z "$grub_kernel" ]; then + # Upstream make binrpm-pkg kernels don't register with grubby; find + # the vmlinuz file and add a boot entry manually. + local vmlinuz + vmlinuz=$(ls /boot/vmlinuz-*"$installed_version"* 2>/dev/null | head -1) + if [ -n "$vmlinuz" ]; then + echo "Adding grubby entry for $vmlinuz" + local initrd="/boot/initramfs-${installed_version}.img" + if [ ! -f "$initrd" ]; then + echo "Generating initramfs at $initrd" + sudo dracut --force "$initrd" "$installed_version" 2>/dev/null \ + || sudo mkinitrd "$initrd" "$installed_version" 2>/dev/null \ + || true + fi + if [ -f "$initrd" ]; then + sudo grubby --add-kernel="$vmlinuz" \ + --initrd="$initrd" \ + --title="Linux $installed_version" \ + --copy-default \ + --make-default + echo "Added and set default: $vmlinuz" + else + echo "WARNING: No initramfs for $installed_version, trying set-default anyway" + sudo grubby --set-default="$vmlinuz" || true + fi + grub_kernel="$vmlinuz" + else + echo "WARNING: No vmlinuz found for version $installed_version" + fi + else + echo "Setting default boot kernel to $grub_kernel" + sudo grubby --set-default="$grub_kernel" + fi + + if [ -n "$grub_kernel" ]; then + echo "Verifying default kernel:" + sudo grubby --default-kernel + fi + return 0 + else + echo "ERROR: Failed to install new kernel" >&2 + return 1 + fi +} + +# Install a given kernel RPM (path passed as argument). +install_specified_kernel_rpm() +{ + local kernel_rpm="$1" + if [ -z "$kernel_rpm" ]; then + echo "ERROR: install_specified_kernel_rpm requires a kernel RPM path" >&2 + return 1 + fi + echo "Installing kernel RPM: $(basename "$kernel_rpm")" + install_kernel_rpm "$kernel_rpm" +} + +# AL2023 core repos ship no qemu system emulator (only qemu-img), and SPAL also +# does not ship it. So build a minimal x86_64-softmmu QEMU from source via the +# vendored build-qemu.sh (--enable-kvm, installs to /opt/qemu). +# Sets and exports QEMU for run_tests.sh. +install_qemu() +{ + QEMU_BIN="" + if command -v qemu-system-x86_64 >/dev/null 2>&1; then + QEMU_BIN="$(command -v qemu-system-x86_64)" + echo "QEMU already present: $QEMU_BIN" + else + echo "No distro qemu-system-x86_64 on AL2023; building QEMU from source ..." + sudo -E QEMU_VERSION="${QEMU_VERSION:-9.2.0}" bash "${SOURCE_DIR}/build-qemu.sh" + [ -x /opt/qemu/bin/qemu-system-x86_64 ] && QEMU_BIN=/opt/qemu/bin/qemu-system-x86_64 + fi + if [ -z "$QEMU_BIN" ] || [ ! -x "$QEMU_BIN" ]; then + echo "ERROR: no usable qemu-system-x86_64 after build" >&2 + return 1 + fi + export QEMU="$QEMU_BIN" + export PATH="$(dirname "$QEMU_BIN"):$PATH" + echo "Using QEMU: $QEMU ($("$QEMU" --version | head -1))" + return 0 +} + +# Hardware gate: this test profile targets x86_64 with Intel VT-x (VMX) and a +# usable /dev/kvm -- the only configuration AWS EC2 nested virtualization +# supports (Intel C8i/M8i/R8i/C7i/... families). Fail fast and clearly on +# anything else, before the expensive QEMU build. +verify_hardware() +{ + echo "=== Hardware gate ===" + local arch; arch="$(uname -m)" + if [ "$arch" != "x86_64" ]; then + echo "ERROR: unsupported architecture '$arch'." >&2 + echo " This kvm-unit-tests profile supports x86_64 (Intel) only. EC2 nested" >&2 + echo " virtualization is Intel-VT-x only (C8i/M8i/R8i/C7i/... families)." >&2 + echo " aarch64/Graviton and AMD KVM require a bare-metal instance plus an arch-" >&2 + echo " generalized qemu build (aarch64-softmmu / SVM test set), not yet implemented." >&2 + return 1 + fi + if [ ! -e /dev/kvm ]; then + echo "ERROR: /dev/kvm is missing -- hardware virtualization is not available." >&2 + echo " Launch a nested-virt-capable Intel family (C8i/C7i/...) with" >&2 + echo " CpuOptions={NestedVirtualization:enabled}, or use a bare-metal host." >&2 + return 1 + fi + if grep -qw vmx /proc/cpuinfo; then + echo "OK: x86_64 + Intel VT-x (vmx) + /dev/kvm present ($(ls -l /dev/kvm))." + return 0 + elif grep -qw svm /proc/cpuinfo; then + echo "ERROR: AMD SVM detected, but this profile targets Intel VMX." >&2 + echo " The default subset exercises Intel vmx_* tests. Running on AMD needs the" >&2 + echo " SVM test set on a bare-metal AMD host (EC2 nested virt is Intel-only)." >&2 + return 1 + else + echo "ERROR: no CPU virtualization flag (vmx/svm) exposed to the guest CPU." >&2 + return 1 + fi +} + +# Confirm nested virtualization actually reached the guest. This doubles as a +# regression check on the EC2 nested-virt enablement itself. +verify_nested_virt() +{ + echo "=== Verifying nested virtualization is available ===" + if [ ! -e /dev/kvm ]; then + echo "ERROR: /dev/kvm is missing -- nested virtualization is NOT enabled on this instance." >&2 + echo " Ensure the VM uses a supported family (c8i/c7i/...) so launch_vm.py sets" >&2 + echo " CpuOptions={NestedVirtualization: enabled}." >&2 + return 1 + fi + echo "/dev/kvm present:" + ls -l /dev/kvm + grep -m1 -oE 'vmx|svm' /proc/cpuinfo | head -1 || { + echo "ERROR: no vmx/svm virtualization flag exposed to the guest CPU." >&2 + return 1 + } + echo "Virtualization CPU flag exposed to guest: OK" + return 0 +} + +# Fetch kvm-unit-tests (primary repo, then kernel.org mirror as fallback). +fetch_kvm_unit_tests() +{ + if [ -d "${KVMUT_DIR}/.git" ]; then + echo "kvm-unit-tests already checked out at ${KVMUT_DIR}" + return 0 + fi + echo "=== Cloning kvm-unit-tests ===" + if ! git clone "${KVMUT_REPO}" "${KVMUT_DIR}"; then + echo "Primary clone failed, trying mirror ${KVMUT_REPO_MIRROR}" + git clone "${KVMUT_REPO_MIRROR}" "${KVMUT_DIR}" + fi + if [ -n "${KVMUT_REF}" ]; then + echo "Checking out pinned ref ${KVMUT_REF}" + git -C "${KVMUT_DIR}" checkout --detach "${KVMUT_REF}" + fi + ( cd "${KVMUT_DIR}" && git submodule update --init >/dev/null 2>&1 || true ) + echo "kvm-unit-tests revision: $(git -C "${KVMUT_DIR}" rev-parse --short HEAD)" +} + +# Configure + build the test binaries. +build_kvm_unit_tests() +{ + echo "=== Building kvm-unit-tests ===" + ( cd "${KVMUT_DIR}" && ./configure && make -j"$(nproc)" ) +} + +# Run the suite, tee-ing full output to a log. Uses standalone tests so a single +# invocation runs the whole set. Returns run_tests.sh's own exit code. +run_kvm_unit_tests() +{ + local logfile="$1" + echo "=== Running kvm-unit-tests (accel=kvm) ===" + local args=(-v) + [ -n "${KVMUT_GROUP}" ] && args+=(-g "${KVMUT_GROUP}") + # Optional explicit subset of test names (positional args to run_tests.sh). + local tests=() + [ -n "${KVMUT_TESTS}" ] && read -r -a tests <<< "${KVMUT_TESTS}" + [ ${#tests[@]} -gt 0 ] && echo "Running subset: ${tests[*]}" + # ACCEL=kvm forces hardware acceleration; the run fails loudly if KVM is + # unavailable rather than silently falling back to slow TCG emulation. + ( cd "${KVMUT_DIR}" && ACCEL=kvm ./run_tests.sh "${args[@]}" "${tests[@]}" ) 2>&1 | tee "$logfile" + return "${PIPESTATUS[0]}" +} + +# Parse run_tests.sh output into PASS/FAIL/SKIP counts and a per-test CSV. +# Writes: +# result.txt human summary +# kvm-unit-tests.csv machine-readable per-test rows for KCIDB ingestion +# Sets globals: KVMUT_PASS KVMUT_FAIL KVMUT_SKIP KVMUT_REAL_FAIL +summarize_kvm_unit_tests() +{ + local logfile="$1" + local csv="$2" + local kver arch clean + kver="$(uname -r)" + arch="$(uname -m)" + + # run_tests.sh emits ANSI colour codes (e.g. \e[32mPASS\e[0m), so strip them + # to a clean copy before matching line-anchored PASS/FAIL/SKIP. + clean="$(mktemp)" + sed -E 's/\x1b\[[0-9;]*m//g' "$logfile" >"$clean" + + KVMUT_PASS=$(grep -cE '^PASS ' "$clean" || true) + KVMUT_FAIL=$(grep -cE '^FAIL ' "$clean" || true) + KVMUT_SKIP=$(grep -cE '^SKIP ' "$clean" || true) + + # Per-test CSV, KCIDB-friendly: test_name,status,kernel_version,arch + echo "test_name,status,kernel_version,arch" >"$csv" + # Guard against pipefail when there are zero matches. + grep -E '^(PASS|FAIL|SKIP) ' "$clean" | while read -r status name _rest; do + echo "${name},${status},${kver},${arch}" >>"$csv" + done || true + + # Count "real" failures excluding the ignore list. + KVMUT_REAL_FAIL=0 + local name + while read -r _status name _rest; do + [ -z "$name" ] && continue + local ignored=0 + for ig in ${KVMUT_IGNORE_FAILURES}; do + [ "$name" = "$ig" ] && ignored=1 && break + done + [ "$ignored" -eq 0 ] && KVMUT_REAL_FAIL=$((KVMUT_REAL_FAIL + 1)) + done < <(grep -E '^FAIL ' "$clean" || true) + + rm -f "$clean" + echo "kvm-unit-tests summary: PASS=${KVMUT_PASS} FAIL=${KVMUT_FAIL} SKIP=${KVMUT_SKIP} (real failures after ignore list: ${KVMUT_REAL_FAIL})" +} diff --git a/vm-tests/kvm-unit-tests/dependencies.txt b/vm-tests/kvm-unit-tests/dependencies.txt new file mode 100644 index 0000000..3f96be8 --- /dev/null +++ b/vm-tests/kvm-unit-tests/dependencies.txt @@ -0,0 +1,7 @@ +# Packages required to build and run kvm-unit-tests. +# QEMU is installed separately by install_qemu() in common_lib.sh because the +# package name varies across distros/versions. +git +gcc +make +binutils diff --git a/vm-tests/kvm-unit-tests/external_requirements.json b/vm-tests/kvm-unit-tests/external_requirements.json new file mode 100644 index 0000000..fd217a4 --- /dev/null +++ b/vm-tests/kvm-unit-tests/external_requirements.json @@ -0,0 +1,4 @@ +{ + "kernel-rpms/src": false, + "kernel-rpms/binary": true +} diff --git a/vm-tests/kvm-unit-tests/run-01-install-kernel.sh b/vm-tests/kvm-unit-tests/run-01-install-kernel.sh new file mode 100755 index 0000000..8f542c5 --- /dev/null +++ b/vm-tests/kvm-unit-tests/run-01-install-kernel.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# First stage: install the target kernel from the pipeline's shared kernel-rpms +# area and boot into it, so run-02 exercises kvm-unit-tests against the kernel +# under test rather than the AMI default. The kernel-management flow is reused +# from unixbench-kernel-regression. After this script the client reboots (a +# second run-*.sh follows), bringing up the installed kernel. + +set -euxo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# Record the kernel we booted from so run-02 can confirm the switch. +kernel_before="$(get_running_kernel)" +echo "Kernel before installation: $kernel_before" +save_kernel_version "$kernel_before" "$KERNEL_FILE" + +# Install the target kernel (lowest version in the shared kernel-rpms area). +# If no kernel RPMs are provided (e.g. a standalone run without the pipeline's +# S3 layout), keep the current kernel and let run-02 test that instead. +if first_kernel="$(get_first_kernel_rpm_from_dir)"; then + install_specified_kernel_rpm "$first_kernel" + echo "Installed target kernel; the client reboots into it before run-02." +else + echo "No kernel RPMs found in S3; keeping the current kernel for run-02." +fi + +# Re-execution continues in run-02-kvm-unit-tests.sh after the reboot. diff --git a/vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh b/vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh new file mode 100755 index 0000000..6a6efc2 --- /dev/null +++ b/vm-tests/kvm-unit-tests/run-02-kvm-unit-tests.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Authors: Norbert Manthey +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Run the upstream kvm-unit-tests suite against the guest KVM, exercising the +# kernel's virtualization stack on an EC2 instance with nested virtualization +# enabled. Second stage: runs after run-01 installed the target kernel and the +# client rebooted into it. + +set -euxo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SOURCE_DIR}/common_lib.sh" + +# 0. Confirm we booted the target kernel that run-01 installed. When run-01 did +# not install one (no kernel RPMs provided), test the current kernel. +if [ -f "${KERNEL_FILE}" ]; then + kernel_before="$(load_kernel_version "${KERNEL_FILE}")" + kernel_now="$(get_running_kernel)" + echo "Kernel before run-01 install: ${kernel_before}; now running: ${kernel_now}" + assert_kernel_changed "${kernel_before}" "${kernel_now}" +else + echo "No recorded pre-install kernel; testing the current kernel ($(get_running_kernel))." +fi + +# 1. Hardware gate -- fail fast and clearly on wrong arch / no KVM, before +# installing anything or building QEMU. +verify_hardware || exit 1 + +# 2. Dependencies (build toolchain + QEMU) +install_test_dependencies +install_qemu + +# 3. Confirm nested virt actually reached this guest (regression check). +verify_nested_virt + +# 4. Fetch + build the suite. +fetch_kvm_unit_tests +build_kvm_unit_tests + +# 5. Run and capture output. +LOG="${PWD}/kvm-unit-tests.log" +set +e +run_kvm_unit_tests "$LOG" +RUN_RC=$? +set -e + +# 6. Summarise into a results file + CSV for KCIDB. The `results_` prefix makes +# test-vm-client.sh upload both from the test dir (it globs results_* and +# benchmark-*.csv). The authoritative SUCCESS/FAILED verdict is written by +# the client from this script's exit code. +summarize_kvm_unit_tests "$LOG" "${PWD}/results_kvm-unit-tests.csv" + +{ + echo "kvm-unit-tests result" + echo "kernel: $(uname -r) arch: $(uname -m)" + echo "PASS=${KVMUT_PASS} FAIL=${KVMUT_FAIL} SKIP=${KVMUT_SKIP}" + echo "real_failures=${KVMUT_REAL_FAIL} (ignore list: ${KVMUT_IGNORE_FAILURES:-})" + echo "run_tests.sh exit code: ${RUN_RC}" +} >"${PWD}/results_kvm-unit-tests.txt" +cat "${PWD}/results_kvm-unit-tests.txt" + +echo "Done executing kvm-unit-tests after $SECONDS seconds" + +# 7. Decide pass/fail. +# Observe-only mode reports but never fails the pipeline -- used while the +# stable subset of tests under nested virt is still being characterised. +if [ "${KVMUT_OBSERVE_ONLY}" = "true" ]; then + echo "OBSERVE-ONLY mode: not gating on failures (real_failures=${KVMUT_REAL_FAIL})." + exit 0 +fi + +if [ "${KVMUT_REAL_FAIL}" -gt 0 ]; then + echo "ERROR: ${KVMUT_REAL_FAIL} kvm-unit-tests failed (after ignore list)." >&2 + exit 1 +fi + +echo "kvm-unit-tests passed: no unexpected failures." +exit 0