diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 5437a1a882..fd456dffb6 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -144,6 +144,13 @@ the `release-tag-v1` qualification profile. Failed qualification prevents stable publication but still allows pre-release artifacts to publish with the failure recorded. +For tmachine K3s conformance failures, download the job's +`tmachine-diagnostics-*` artifact. It contains diagnostics fetched before the VM +exits: forwarder restart counts and start-limit settings, K3s and forwarder +journals, listener state, gateway Pod identity/readiness, endpoint state, and +current/previous gateway logs. A readiness failure means scenarios did not run; +use the collector output to identify the failing layer before rerunning. + View logs for a specific run: ```bash diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 089e73a242..58127cd47b 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -86,3 +86,12 @@ jobs: INSTALLER: ${{ matrix.installer }} TESTSUITE: ${{ matrix.testsuite }} run: nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}" + + - name: Upload tmachine diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }} + path: artifacts/tmachine-diagnostics + if-no-files-found: ignore + retention-days: 7 diff --git a/CI.md b/CI.md index c5a19e1f08..c0fd42408e 100644 --- a/CI.md +++ b/CI.md @@ -105,6 +105,28 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. +### K3s boot readiness and diagnostics + +The K3s installer checks readiness during provisioning and again after the automated +test VM boots, including boots from cached installer disks. The checks wait for the +API, nodes, and gateway StatefulSet, then require `openshell status --output json` +to report the registered `tmachine` gateway as connected through the forwarder. Readiness +failures stop the suite before its scenarios run; scenario timeouts are unchanged. +The loopback forwarder retries every five seconds without a systemd start limit +so it can recover after temporary K3s API failures. + +Interactive `shell` suites skip test-boot preparation so you can inspect a gateway +that failed during boot. They still run the installer checks when provisioning. + +Installation, readiness, and conformance failures collect bounded K3s and +forwarder journals, service state, listener state, Pod identity and container +status, endpoints, events, and current and previous gateway logs before the VM +exits. The collector omits Secrets, kubeconfigs, environment dumps, and full Pod +specifications and redacts common credential fields. Local runs save these to +`artifacts/tmachine-diagnostics`; CI uploads them as `tmachine-diagnostics-*` +artifacts even when the test step fails. Check these alongside the Ansible error +to distinguish forwarding failures from gateway restarts or readiness failures. + ### Run only the policy advisor conformance tests Manually dispatch `Integration Tests` on the candidate branch with an diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index dd8ce2b592..fee1eac41c 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -106,6 +106,11 @@ var: openshell_gateway_logs.stdout_lines when: conformance_result.rc != 0 + - name: Collect K3s conformance diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + when: conformance_result.rc != 0 + - name: Require OpenShell conformance success ansible.builtin.assert: that: diff --git a/tests/ansible/playbooks/openshell-k3s-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ready.yaml new file mode 100644 index 0000000000..3a0c7cc61d --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ready.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check OpenShell K3s readiness after boot + hosts: all + gather_facts: false + roles: + - openshell_k3s_ready diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index dd00aa4bda..03e21fc077 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -10,166 +10,174 @@ - name: Wait for SSH ansible.builtin.wait_for_connection: - - name: Install OpenShell CLI - ansible.builtin.copy: - src: "{{ openshell_cli_binary }}" - dest: /usr/local/bin/openshell - mode: "0755" - - - name: Create OpenShell artifact directory - ansible.builtin.file: - path: /var/lib/openshell/artifacts - state: directory - mode: "0700" - - - name: Copy OpenShell images - ansible.builtin.copy: - src: "{{ item.src }}" - dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" - mode: "0600" - loop: - - name: gateway - src: "{{ openshell_gateway_image }}" - - name: sandbox - src: "{{ openshell_sandbox_image }}" - - name: supervisor - src: "{{ openshell_supervisor_image }}" - - - name: Import OpenShell images into K3s - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - ctr - - --namespace - - k8s.io - - images - - import - - "/var/lib/openshell/artifacts/{{ item }}.tar" - loop: - - gateway - - sandbox - - supervisor - - - name: Copy OpenShell Helm chart - ansible.builtin.copy: - src: "{{ openshell_helm_chart }}" - dest: /var/lib/openshell/artifacts/helm-chart.tgz - mode: "0600" - - - name: Write OpenShell Helm values - ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values.yaml - mode: "0600" - content: | - global: - image: - registry: "" - gateway: - image: - repository: openshell/gateway - tag: tmachine - pullPolicy: Never - sandboxRuntime: - image: - repository: openshell/sandbox - tag: tmachine - pullPolicy: never - supervisor: - image: - repository: openshell/supervisor - tag: tmachine - pullPolicy: never - networkPolicy: + - name: Provision the K3s gateway + block: + - name: Install OpenShell CLI + ansible.builtin.copy: + src: "{{ openshell_cli_binary }}" + dest: /usr/local/bin/openshell + mode: "0755" + + - name: Create OpenShell artifact directory + ansible.builtin.file: + path: /var/lib/openshell/artifacts + state: directory + mode: "0700" + + - name: Copy OpenShell images + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" + mode: "0600" + loop: + - name: gateway + src: "{{ openshell_gateway_image }}" + - name: sandbox + src: "{{ openshell_sandbox_image }}" + - name: supervisor + src: "{{ openshell_supervisor_image }}" + + - name: Import OpenShell images into K3s + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - ctr + - --namespace + - k8s.io + - images + - import + - "/var/lib/openshell/artifacts/{{ item }}.tar" + loop: + - gateway + - sandbox + - supervisor + + - name: Copy OpenShell Helm chart + ansible.builtin.copy: + src: "{{ openshell_helm_chart }}" + dest: /var/lib/openshell/artifacts/helm-chart.tgz + mode: "0600" + + - name: Write OpenShell Helm values + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values.yaml + mode: "0600" + content: | + global: + image: + registry: "" + gateway: + image: + repository: openshell/gateway + tag: tmachine + pullPolicy: Never + sandboxRuntime: + image: + repository: openshell/sandbox + tag: tmachine + pullPolicy: never + supervisor: + image: + repository: openshell/supervisor + tag: tmachine + pullPolicy: never + networkPolicy: + enabled: true + server: + auth: + allowUnauthenticatedUsers: true + disableTls: true + telemetryEnabled: false + + - name: Install Agent Sandbox + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - apply + - --filename + - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" + + - name: Wait for Agent Sandbox CRD + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - wait + - --for=condition=Established + - crd/sandboxes.agents.x-k8s.io + - --timeout=120s + changed_when: false + + - name: Wait for Agent Sandbox controller + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - agent-sandbox-system + - rollout + - status + - deployment/agent-sandbox-controller + - --timeout=300s + changed_when: false + + - name: Install OpenShell Helm chart + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - install + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --create-namespace + - --values + - /var/lib/openshell/artifacts/values.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + + - name: Install gateway port-forward service + ansible.builtin.copy: + dest: /etc/systemd/system/openshell-k3s-port-forward.service + mode: "0644" + content: | + [Unit] + Description=OpenShell K3s gateway port forward + After=k3s.service + Requires=k3s.service + # API recovery must not leave this unit permanently rate-limited. + StartLimitIntervalSec=0 + + [Service] + ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target + + - name: Start gateway port-forward service + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + daemon_reload: true enabled: true - server: - auth: - allowUnauthenticatedUsers: true - disableTls: true - telemetryEnabled: false - - - name: Install Agent Sandbox - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - apply - - --filename - - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" - - - name: Wait for Agent Sandbox CRD - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - wait - - --for=condition=Established - - crd/sandboxes.agents.x-k8s.io - - --timeout=120s - changed_when: false - - - name: Wait for Agent Sandbox controller - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - --namespace - - agent-sandbox-system - - rollout - - status - - deployment/agent-sandbox-controller - - --timeout=300s - changed_when: false - - - name: Install OpenShell Helm chart - ansible.builtin.command: - argv: - - /usr/local/bin/helm - - install - - openshell - - /var/lib/openshell/artifacts/helm-chart.tgz - - --namespace - - openshell - - --create-namespace - - --values - - /var/lib/openshell/artifacts/values.yaml - - --wait - - --timeout=5m - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - - name: Install gateway port-forward service - ansible.builtin.copy: - dest: /etc/systemd/system/openshell-k3s-port-forward.service - mode: "0644" - content: | - [Unit] - Description=OpenShell K3s gateway port forward - After=k3s.service - Requires=k3s.service - - [Service] - ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 - Restart=always - RestartSec=1 - - [Install] - WantedBy=multi-user.target - - - name: Start gateway port-forward service - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - daemon_reload: true - enabled: true - state: started - - - name: Wait for OpenShell gateway - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 17670 - timeout: 60 + state: started + + rescue: + - name: Collect K3s installation diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + + - name: Fail K3s installation + ansible.builtin.fail: + msg: OpenShell K3s installation failed; see K3s diagnostics - name: Register OpenShell gateway for test client hosts: all gather_facts: false roles: - openshell_client + - openshell_k3s_ready diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh new file mode 100644 index 0000000000..6ab7d05360 --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# No kubeconfig, Secrets, environment dumps, or complete Pod specifications. +set -u +umask 077 +mkdir -p /var/lib/openshell/diagnostics +output=/var/lib/openshell/diagnostics/k3s.txt + +collect() { + printf '\n### %s\n' "$1" + shift + # Bound both stalled commands and unusually large log streams. + timeout 10s "$@" 2>&1 | tail -c 262144 + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" +} + +{ + date --utc --iso-8601=seconds + collect 'Service state' systemctl show k3s.service openshell-k3s-port-forward.service \ + -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \ + -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ + -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec + collect 'Forwarder unit' systemctl cat openshell-k3s-port-forward.service + collect 'K3s and forwarder boot journals' journalctl -b --no-pager --lines=500 \ + -u k3s.service -u openshell-k3s-port-forward.service + collect 'Gateway listener' ss -ltnp 'sport = :17670' + collect 'K3s version' /usr/local/bin/k3s --version + collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz + collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide + collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' + collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \ + -o 'custom-columns=NAME:.metadata.name,SELECTOR:.spec.selector,PORTS:.spec.ports' + collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ + -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' + collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp + collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps + collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps + collect 'Memory' free -m + collect 'Disk' df -h / /var/lib/rancher/k3s +} | sed -E \ + -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \ + -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' \ + > "$output" +cat "$output" diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml new file mode 100644 index 0000000000..e1ee0ba622 --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check for K3s + become: true + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_diagnostics_binary + failed_when: false + +- name: Preserve K3s diagnostics before the VM exits + when: k3s_diagnostics_binary.stat.exists | default(false) + become: true + block: + - name: Collect bounded K3s diagnostics + ansible.builtin.script: collect.sh + register: k3s_diagnostics_collection + changed_when: false + failed_when: false + + - name: Show K3s diagnostics + ansible.builtin.debug: + var: k3s_diagnostics_collection.stdout_lines + + - name: Fetch K3s diagnostics + ansible.builtin.fetch: + src: /var/lib/openshell/diagnostics/k3s.txt + dest: "{{ lookup('env', 'TMACHINE_DIAGNOSTICS_DIR') | default(role_path + '/../../../../artifacts/tmachine-diagnostics', true) }}/{{ inventory_hostname }}/k3s.txt" + flat: true + failed_when: false diff --git a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml new file mode 100644 index 0000000000..301cb9ea58 --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Wait for the installed K3s gateway + block: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + timeout: 120 + + - name: Wait for the K3s API + become: true + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz + register: k3s_api_ready + changed_when: false + until: k3s_api_ready.rc == 0 + retries: 30 + delay: 2 + + - name: Wait for K3s nodes + become: true + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=130s wait nodes --all --for=condition=Ready --timeout=120s + changed_when: false + + - name: Wait for the gateway StatefulSet + become: true + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=190s --namespace openshell rollout status statefulset/openshell --timeout=180s + changed_when: false + + # A connected CLI proves the forwarder works. Allow time for it to recover + # without restarting or resetting the unit here. + - name: Wait for the registered gateway to answer the CLI + become: true + become_user: tmachine + ansible.builtin.command: timeout 10s openshell status --output json + register: k3s_gateway_ready + changed_when: false + until: >- + k3s_gateway_ready.rc == 0 and + (k3s_gateway_ready.stdout | from_json).status == 'connected' and + (k3s_gateway_ready.stdout | from_json).gateway == 'tmachine' and + (k3s_gateway_ready.stdout | from_json).server == 'http://127.0.0.1:17670' + retries: 24 + delay: 5 + rescue: + - name: Collect K3s readiness diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + + - name: Fail K3s boot preparation + ansible.builtin.fail: + msg: OpenShell K3s gateway did not become ready before conformance; see K3s diagnostics diff --git a/tests/config.nix b/tests/config.nix index 26d27b712b..28bedc8f3f 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -85,6 +85,7 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; + prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; inputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; @@ -213,6 +214,7 @@ let root=$(git rev-parse --show-toplevel) cd "$root/tests" export ANSIBLE_CONFIG="$PWD/ansible/ansible.cfg" + export TMACHINE_DIAGNOSTICS_DIR="$root/artifacts/tmachine-diagnostics" exec ${tmachine}/bin/tmachine --config ${config} "$@" ''; }; diff --git a/tests/tmachine/src/config.rs b/tests/tmachine/src/config.rs index bc19cbbb1f..44a614538b 100644 --- a/tests/tmachine/src/config.rs +++ b/tests/tmachine/src/config.rs @@ -40,6 +40,34 @@ pub struct Installer { pub use_galaxy: bool, pub playbooks: Vec, pub inputs: BTreeMap, + /// Readiness checks run on non-interactive test boots, including cached installations. + #[serde(default)] + pub prepare_playbooks: Vec, +} + +#[cfg(test)] +mod tests { + use super::Installer; + + #[test] + fn existing_installers_need_no_boot_preparation() { + let installer: Installer = + serde_saphyr::from_str("name: none\nuse_galaxy: false\nplaybooks: []\ninputs: {}\n") + .unwrap(); + assert!(installer.prepare_playbooks.is_empty()); + } + + #[test] + fn installer_can_request_boot_preparation() { + let installer: Installer = serde_saphyr::from_str( + "name: k3s\nuse_galaxy: false\nplaybooks: []\ninputs: {}\nprepare_playbooks: [ready.yaml]\n", + ) + .unwrap(); + assert_eq!( + installer.prepare_playbooks, + [std::path::PathBuf::from("ready.yaml")] + ); + } } #[derive(Clone, Deserialize)] diff --git a/tests/tmachine/src/qemu/test.rs b/tests/tmachine/src/qemu/test.rs index ac8a8bdfc0..0dd8f9888c 100644 --- a/tests/tmachine/src/qemu/test.rs +++ b/tests/tmachine/src/qemu/test.rs @@ -26,6 +26,11 @@ pub async fn test( let image = QemuImage::create(&install_disk, test_disk).await; let vm = QemuVm::start(&image).await; + // Installation readiness does not survive the shutdown of the cached VM. + // Keep the interactive shell accessible when diagnosing boot failures. + if !testsuite.interactive { + run_playbooks(&installer.prepare_playbooks, &installer.inputs).await?; + } run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?; if testsuite.interactive {