From 09807b5c0dadf90c04e6d44bea886784c5f4eb56 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 1 Oct 2026 14:54:54 -0700 Subject: [PATCH 1/6] test(tmachine): add K3s PostgreSQL and mTLS conformance installer Signed-off-by: Matthew Grossman --- .agents/skills/helm-dev-environment/SKILL.md | 16 + .github/workflows/branch-e2e.yml | 1 + .github/workflows/integration-runner.yml | 9 + .github/workflows/integration-test.yml | 1 + skills/debug-openshell-cluster/SKILL.md | 9 + tests/README.md | 58 +++ tests/ansible/playbooks/conformance/cli.yaml | 15 +- .../playbooks/openshell-k3s-ready.yaml | 10 + tests/ansible/playbooks/openshell-k3s.yaml | 373 ++++++++++-------- .../files/collect.sh | 50 +++ .../openshell_k3s_diagnostics/tasks/main.yaml | 31 ++ .../roles/openshell_k3s_ready/tasks/main.yaml | 116 ++++++ tests/config.nix | 17 + tests/tmachine/src/config.rs | 28 ++ tests/tmachine/src/qemu/test.rs | 1 + 15 files changed, 571 insertions(+), 164 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/ansible/playbooks/openshell-k3s-ready.yaml create mode 100644 tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh create mode 100644 tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml create mode 100644 tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index cb3023f476..be198f04c7 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -296,6 +296,22 @@ KUBECONFIG=kubeconfig helm upgrade openshell deploy/helm/openshell \ Use the IP that pods in that cluster use to reach listeners on the test host. +### tmachine PostgreSQL/mTLS conformance + +For a disposable VM using staged candidate artifacts, run: + +```shell +nix run .#tmachine -- test ubuntu-k3s k3s-ha-tls conformance +``` + +This installer uses three gateway replicas on one K3s node, the pinned e2e +PostgreSQL fixture and chart-generated mTLS. It registers the guest CLI at +`https://localhost:17670` and checks an authenticated API after every boot. +The existing `k3s` installer remains the SQLite/plaintext baseline. See +`tests/README.md` for artifact prerequisites and interactive guest access. +This lane runs ordinary conformance; scale/owner-loss/rollout scenarios and +node-level HA qualification require separate coverage. + ### BackendTLSPolicy (end-to-end TLS) To enable end-to-end TLS between the Gateway proxy and the gateway pod, add diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 49cf199a3c..643e5fc6e9 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -225,6 +225,7 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 768879c7b8..c7dbd8c58a 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -85,4 +85,13 @@ jobs: ENVIRONMENT: ${{ matrix.environment }} INSTALLER: ${{ matrix.installer }} TESTSUITE: ${{ matrix.testsuite }} + TMACHINE_DIAGNOSTICS_DIR: ${{ github.workspace }}/artifacts/tmachine-diagnostics run: nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}" + + - name: Upload tmachine diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }} + path: artifacts/tmachine-diagnostics + if-no-files-found: ignore diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index bc91802882..ec8521cc61 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -24,6 +24,7 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"} ] diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 6382428946..51ca49e714 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -447,6 +447,15 @@ stores encrypted credential envelopes in the OpenShell database. For mentioning `server.credentialDrivers` means the values selected multiple external credential backends. +For a disposable single-node PostgreSQL/mTLS test deployment, check that the +CLI registration uses an HTTPS endpoint and that the named gateway's mTLS +directory contains the chart client certificate and CA. An open port-forward +or successful `gateway add` alone does not establish API readiness: require a +successful `openshell sandbox list --output json` as well. After the cluster +reboots, wait for PostgreSQL, the controller and all gateway replicas before +diagnosing sandbox operations. Collect Kubernetes gateway logs rather than a +host `openshell-gateway.service` journal for a Helm deployment. + For HA or PostgreSQL-backed installs, also check the external database Secret referenced by `server.externalDbSecret` and the PostgreSQL workload when it is deployed in-cluster: diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..83ddd84746 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,58 @@ + + +# tmachine tests + +`tmachine` boots a disposable QEMU guest, prepares a runtime environment, +installs candidate artifacts, and runs a test archive. Its configuration lives +in `tests/config.nix`. The guest uses four CPUs and 4 GiB of RAM; tests run on +the host's native architecture with HVF on Apple Silicon or KVM on Linux. + +## K3s conformance + +Stage the native musl CLI, candidate gateway/sandbox/supervisor OCI archives, +Helm chart and conformance nextest bundle under `artifacts/`. The existing +`build-artifacts` flake app builds these locally; CI's +`prepare-integration-inputs.yml` stages them from the same source revision. + +The `ubuntu-k3s` environment installs K3s, Helm and nextest. Choose the basic +single-replica SQLite/plaintext installer or the PostgreSQL/mTLS installer: + +```shell +nix run .#tmachine -- test ubuntu-k3s k3s conformance +nix run .#tmachine -- test ubuntu-k3s k3s-ha-tls conformance +``` + +`k3s-ha-tls` installs three gateway replicas as a Deployment, the pinned +PostgreSQL fixture from `e2e/kubernetes/postgres-fixture.yaml`, and the chart's +built-in PKI. The gateway and peer connections use TLS; the CLI uses the chart +client certificate and `https://localhost:17670`. Unauthenticated users are +disabled. PostgreSQL uses disposable storage and test-only credentials, with +an unencrypted database connection inside the isolated guest. The fixture +image and Agent Sandbox controller are pulled by the guest during setup. + +Installer `prepare_playbooks` run before every test suite, including boots +from a cached installation. K3s preparation waits for the node, PostgreSQL +when present, controller, gateway workload and a successful sandbox-list RPC. +Client certificate files are installed with mode `0600`. A restarting systemd +port-forward exposes the gateway only on guest loopback. Readiness and +conformance failures print cluster inventory, events and bounded workload and +service logs without reading Secret contents. + +For an interactive guest using the same installer: + +```shell +nix run .#tmachine -- test ubuntu-k3s k3s-ha-tls shell +``` + +Inside the guest, use `sudo k3s kubectl` for cluster inspection. Conformance +runs as the `tmachine` user using its registered gateway and certificate +bundle. Each test boot gets a disposable overlay of the installed disk. + +The branch conformance matrix and manual Integration Test workflow include +the PostgreSQL/mTLS tuple. This setup runs the existing portable conformance +scenarios. HA fault injection and migration of #3825's scale/owner-loss/rollout +scenarios are follow-up work. Three replicas on one K3s node do not establish +node-level HA, database HA, ingress behavior or uninterrupted stream recovery. diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index fb3220b07e..4fff19b778 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -85,6 +85,17 @@ var: conformance_result when: conformance_result.rc != 0 + - name: Detect K3s for conformance diagnostics + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: conformance_k3s + when: conformance_result.rc != 0 + + - name: Collect Kubernetes conformance diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + when: conformance_result.rc != 0 and conformance_k3s.stat.exists + - name: Read OpenShell gateway logs become: true ansible.builtin.command: @@ -98,12 +109,12 @@ register: openshell_gateway_logs changed_when: false failed_when: false - when: conformance_result.rc != 0 + when: conformance_result.rc != 0 and not conformance_k3s.stat.exists - name: Show OpenShell gateway logs ansible.builtin.debug: var: openshell_gateway_logs.stdout_lines - when: conformance_result.rc != 0 + when: conformance_result.rc != 0 and not conformance_k3s.stat.exists - name: Require OpenShell conformance success ansible.builtin.assert: diff --git a/tests/ansible/playbooks/openshell-k3s-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ready.yaml new file mode 100644 index 0000000000..75fc711dcc --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ready.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Prepare K3s gateway after every test boot + hosts: all + become: true + gather_facts: false + roles: + - openshell_k3s_ready diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index dd00aa4bda..14bf5e4fd2 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -6,170 +6,219 @@ hosts: all become: true gather_facts: false + vars: + k3s_ha: "{{ openshell_k3s_ha | default(false) | bool }}" tasks: - - name: Wait for SSH - ansible.builtin.wait_for_connection: - - - name: Install OpenShell CLI - ansible.builtin.copy: - src: "{{ openshell_cli_binary }}" - dest: /usr/local/bin/openshell - mode: "0755" - - - name: Create OpenShell artifact directory - ansible.builtin.file: - path: /var/lib/openshell/artifacts - state: directory - mode: "0700" - - - name: Copy OpenShell images - ansible.builtin.copy: - src: "{{ item.src }}" - dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" - mode: "0600" - loop: - - name: gateway - src: "{{ openshell_gateway_image }}" - - name: sandbox - src: "{{ openshell_sandbox_image }}" - - name: supervisor - src: "{{ openshell_supervisor_image }}" - - - name: Import OpenShell images into K3s - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - ctr - - --namespace - - k8s.io - - images - - import - - "/var/lib/openshell/artifacts/{{ item }}.tar" - loop: - - gateway - - sandbox - - supervisor - - - name: Copy OpenShell Helm chart - ansible.builtin.copy: - src: "{{ openshell_helm_chart }}" - dest: /var/lib/openshell/artifacts/helm-chart.tgz - mode: "0600" - - - name: Write OpenShell Helm values - ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values.yaml - mode: "0600" - content: | - global: - image: - registry: "" - gateway: - image: - repository: openshell/gateway - tag: tmachine - pullPolicy: Never - sandboxRuntime: - image: - repository: openshell/sandbox - tag: tmachine - pullPolicy: never - supervisor: - image: - repository: openshell/supervisor - tag: tmachine - pullPolicy: never - networkPolicy: + - name: Install K3s gateway and fixtures + block: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Install OpenShell CLI + ansible.builtin.copy: + src: "{{ openshell_cli_binary }}" + dest: /usr/local/bin/openshell + mode: "0755" + + - name: Create OpenShell artifact directory + ansible.builtin.file: + path: /var/lib/openshell/artifacts + state: directory + mode: "0700" + + - name: Copy OpenShell images + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" + mode: "0600" + loop: + - name: gateway + src: "{{ openshell_gateway_image }}" + - name: sandbox + src: "{{ openshell_sandbox_image }}" + - name: supervisor + src: "{{ openshell_supervisor_image }}" + + - name: Import OpenShell images into K3s + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - ctr + - --namespace + - k8s.io + - images + - import + - "/var/lib/openshell/artifacts/{{ item }}.tar" + loop: + - gateway + - sandbox + - supervisor + + - name: Copy OpenShell Helm chart + ansible.builtin.copy: + src: "{{ openshell_helm_chart }}" + dest: /var/lib/openshell/artifacts/helm-chart.tgz + mode: "0600" + + - name: Write OpenShell Helm values + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values.yaml + mode: "0600" + content: | + global: + image: + registry: "" + replicaCount: {{ 3 if k3s_ha else 1 }} + workload: + kind: {{ 'deployment' if k3s_ha else 'statefulset' }} + gateway: + image: + repository: openshell/gateway + tag: tmachine + pullPolicy: Never + sandboxRuntime: + image: + repository: openshell/sandbox + tag: tmachine + pullPolicy: never + supervisor: + image: + repository: openshell/supervisor + tag: tmachine + pullPolicy: never + networkPolicy: + enabled: true + server: + auth: + allowUnauthenticatedUsers: {{ (not k3s_ha) | to_json }} + disableTls: {{ (not k3s_ha) | to_json }} + telemetryEnabled: false + {% if k3s_ha %} + externalDbSecret: openshell-e2e-postgres-credentials + peer: + allowInsecureTransport: false + credentialDrivers: + kubernetesSecrets: + enabled: true + namespace: openshell + {% endif %} + + - name: Install PostgreSQL fixture + when: k3s_ha + block: + - name: Create OpenShell namespace + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, create, namespace, openshell] + + - name: Copy pinned PostgreSQL fixture + ansible.builtin.copy: + src: "{{ openshell_postgres_fixture }}" + dest: /var/lib/openshell/artifacts/postgres.yaml + mode: "0600" + + - name: Apply PostgreSQL fixture + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] + + - name: Wait for PostgreSQL + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] + changed_when: false + + - name: Install Agent Sandbox + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - apply + - --filename + - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" + + - name: Wait for Agent Sandbox CRD + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - wait + - --for=condition=Established + - crd/sandboxes.agents.x-k8s.io + - --timeout=120s + changed_when: false + + - name: Wait for Agent Sandbox controller + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - agent-sandbox-system + - rollout + - status + - deployment/agent-sandbox-controller + - --timeout=300s + changed_when: false + + - name: Install OpenShell Helm chart + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - install + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --create-namespace + - --values + - /var/lib/openshell/artifacts/values.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + + - name: Install gateway port-forward service + ansible.builtin.copy: + dest: /etc/systemd/system/openshell-k3s-port-forward.service + mode: "0644" + content: | + [Unit] + Description=OpenShell K3s gateway port forward + After=k3s.service + Requires=k3s.service + StartLimitIntervalSec=0 + + [Service] + ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target + + - name: Start gateway port-forward service + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + daemon_reload: true enabled: true - server: - auth: - allowUnauthenticatedUsers: true - disableTls: true - telemetryEnabled: false - - - name: Install Agent Sandbox - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - apply - - --filename - - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" - - - name: Wait for Agent Sandbox CRD - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - wait - - --for=condition=Established - - crd/sandboxes.agents.x-k8s.io - - --timeout=120s - changed_when: false - - - name: Wait for Agent Sandbox controller - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - --namespace - - agent-sandbox-system - - rollout - - status - - deployment/agent-sandbox-controller - - --timeout=300s - changed_when: false - - - name: Install OpenShell Helm chart - ansible.builtin.command: - argv: - - /usr/local/bin/helm - - install - - openshell - - /var/lib/openshell/artifacts/helm-chart.tgz - - --namespace - - openshell - - --create-namespace - - --values - - /var/lib/openshell/artifacts/values.yaml - - --wait - - --timeout=5m - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - - name: Install gateway port-forward service - ansible.builtin.copy: - dest: /etc/systemd/system/openshell-k3s-port-forward.service - mode: "0644" - content: | - [Unit] - Description=OpenShell K3s gateway port forward - After=k3s.service - Requires=k3s.service - - [Service] - ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 - Restart=always - RestartSec=1 - - [Install] - WantedBy=multi-user.target - - - name: Start gateway port-forward service - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - daemon_reload: true - enabled: true - state: started - - - name: Wait for OpenShell gateway - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 17670 - timeout: 60 - -- name: Register OpenShell gateway for test client + state: started + + - name: Wait for OpenShell gateway + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 17670 + timeout: 60 + + rescue: + - name: Collect installation diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + + - name: Fail K3s installation + ansible.builtin.fail: + msg: K3s gateway installation failed; see the cluster diagnostics above. + +- name: Prepare OpenShell gateway for test client hosts: all + become: true gather_facts: false roles: - - openshell_client + - openshell_k3s_ready diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh new file mode 100644 index 0000000000..1f61c736be --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# No kubeconfig, Secrets, environment dumps, or complete Pod specifications. +set -u +umask 077 +mkdir -p /var/lib/openshell/diagnostics +output=/var/lib/openshell/diagnostics/k3s.txt + +collect() { + printf '\n### %s\n' "$1" + shift + # Bound both stalled commands and unusually large log streams. + timeout 10s "$@" 2>&1 | tail -c 262144 + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" +} + +{ + date --utc --iso-8601=seconds + collect 'Service state' systemctl show k3s.service openshell-k3s-port-forward.service \ + -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \ + -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ + -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec + collect 'Forwarder unit' systemctl cat openshell-k3s-port-forward.service + collect 'K3s and forwarder boot journals' journalctl -b --no-pager --lines=500 \ + -u k3s.service -u openshell-k3s-port-forward.service + collect 'Gateway listener' ss -ltnp 'sport = :17670' + collect 'K3s version' /usr/local/bin/k3s --version + collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz + collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide + collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' + collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \ + -o 'custom-columns=NAME:.metadata.name,SELECTOR:.spec.selector,PORTS:.spec.ports' + collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ + -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' + collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp + collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs -l app.kubernetes.io/name=openshell -c openshell-gateway --max-log-requests=5 --tail=300 --timestamps + collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs -l app.kubernetes.io/name=openshell -c openshell-gateway --max-log-requests=5 --previous --tail=300 --timestamps + collect 'PostgreSQL logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs deployment/openshell-e2e-postgres --tail=100 + collect 'Controller logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs deployment/agent-sandbox-controller --tail=100 + collect 'Memory' free -m + collect 'Disk' df -h / /var/lib/rancher/k3s +} | sed -E \ + -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \ + -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' \ + -e 's#(postgres(ql)?://)[^/@ ]+@#\1[REDACTED]@#gI' \ + > "$output" +cat "$output" diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml new file mode 100644 index 0000000000..e1ee0ba622 --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check for K3s + become: true + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_diagnostics_binary + failed_when: false + +- name: Preserve K3s diagnostics before the VM exits + when: k3s_diagnostics_binary.stat.exists | default(false) + become: true + block: + - name: Collect bounded K3s diagnostics + ansible.builtin.script: collect.sh + register: k3s_diagnostics_collection + changed_when: false + failed_when: false + + - name: Show K3s diagnostics + ansible.builtin.debug: + var: k3s_diagnostics_collection.stdout_lines + + - name: Fetch K3s diagnostics + ansible.builtin.fetch: + src: /var/lib/openshell/diagnostics/k3s.txt + dest: "{{ lookup('env', 'TMACHINE_DIAGNOSTICS_DIR') | default(role_path + '/../../../../artifacts/tmachine-diagnostics', true) }}/{{ inventory_hostname }}/k3s.txt" + flat: true + failed_when: false diff --git a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml new file mode 100644 index 0000000000..05ac879068 --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Wait for SSH + ansible.builtin.wait_for_connection: + +- name: Resolve K3s gateway configuration + ansible.builtin.set_fact: + k3s_ha: "{{ openshell_k3s_ha | default(false) | bool }}" + +- name: Prepare K3s gateway + block: + - name: Wait for K3s API + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --request-timeout=10s, get, nodes, --output, name] + register: k3s_api + until: k3s_api.rc == 0 + retries: 30 + delay: 5 + changed_when: false + + - name: Wait for K3s node + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --request-timeout=190s, wait, --for=condition=Ready, nodes, --all, --timeout=180s] + changed_when: false + + - name: Wait for PostgreSQL after boot + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] + changed_when: false + when: k3s_ha + + - name: Wait for Agent Sandbox controller + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, agent-sandbox-system, rollout, status, deployment/agent-sandbox-controller, --timeout=300s] + changed_when: false + + - name: Wait for gateway workload + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, "{{ 'deployment' if k3s_ha else 'statefulset' }}/openshell", --timeout=300s] + changed_when: false + + - name: Start gateway port-forward service + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + state: started + + - name: Wait for gateway listener + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 17670 + timeout: 120 + + - name: Install mTLS client identity + when: k3s_ha + block: + - name: Read chart client TLS Secret + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] + register: k3s_client_tls + changed_when: false + no_log: true + + - name: Create guest client config directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + loop: + - /home/tmachine/.config + - /home/tmachine/.config/openshell + - /home/tmachine/.config/openshell/gateways + - /home/tmachine/.config/openshell/gateways/tmachine + - /home/tmachine/.config/openshell/gateways/tmachine/mtls + + - name: Write guest client TLS bundle + ansible.builtin.copy: + content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" + dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" + owner: tmachine + group: tmachine + mode: "0600" + loop: [ca.crt, tls.crt, tls.key] + no_log: true + + - name: Register guest gateway + ansible.builtin.include_role: + name: openshell_client + apply: + become: false + vars: + openshell_client_gateway_endpoint: "{{ 'https://localhost:17670' if k3s_ha else 'http://127.0.0.1:17670' }}" + + # gateway add can warn about an unreachable endpoint and still return zero. + # A sandbox-list RPC exercises TLS, authentication and the shared database. + - name: Wait for authenticated gateway API + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, sandbox, list, --output, json] + register: k3s_gateway_api + until: k3s_gateway_api.rc == 0 + retries: 24 + delay: 5 + changed_when: false + + rescue: + - name: Collect K3s diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + + - name: Fail K3s preparation + ansible.builtin.fail: + msg: K3s gateway preparation failed; see the cluster diagnostics above. diff --git a/tests/config.nix b/tests/config.nix index 4bf7488ff8..d8a0a6648d 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -85,6 +85,7 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; + prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; inputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; @@ -94,6 +95,22 @@ let openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; }; } + { + name = "k3s-ha-tls"; + use_galaxy = false; + playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; + prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; + inputs = { + openshell_k3s_ha = "true"; + openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; + agent_sandbox_version = "0.5.0"; + openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; + openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar"; + openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz"; + openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar"; + openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar"; + }; + } { name = "none"; use_galaxy = false; diff --git a/tests/tmachine/src/config.rs b/tests/tmachine/src/config.rs index bc19cbbb1f..6e6abfd009 100644 --- a/tests/tmachine/src/config.rs +++ b/tests/tmachine/src/config.rs @@ -40,6 +40,34 @@ pub struct Installer { pub use_galaxy: bool, pub playbooks: Vec, pub inputs: BTreeMap, + /// Run on every test boot, including when the installed disk is cached. + #[serde(default)] + pub prepare_playbooks: Vec, +} + +#[cfg(test)] +mod tests { + use super::Installer; + + #[test] + fn existing_installers_need_no_boot_preparation() { + let installer: Installer = + serde_saphyr::from_str("name: none\nuse_galaxy: false\nplaybooks: []\ninputs: {}\n") + .unwrap(); + assert!(installer.prepare_playbooks.is_empty()); + } + + #[test] + fn installer_can_request_boot_preparation() { + let installer: Installer = serde_saphyr::from_str( + "name: k3s\nuse_galaxy: false\nplaybooks: []\ninputs: {}\nprepare_playbooks: [ready.yaml]\n", + ) + .unwrap(); + assert_eq!( + installer.prepare_playbooks, + [std::path::PathBuf::from("ready.yaml")] + ); + } } #[derive(Clone, Deserialize)] diff --git a/tests/tmachine/src/qemu/test.rs b/tests/tmachine/src/qemu/test.rs index ac8a8bdfc0..0da66fe088 100644 --- a/tests/tmachine/src/qemu/test.rs +++ b/tests/tmachine/src/qemu/test.rs @@ -26,6 +26,7 @@ pub async fn test( let image = QemuImage::create(&install_disk, test_disk).await; let vm = QemuVm::start(&image).await; + run_playbooks(&installer.prepare_playbooks, &installer.inputs).await?; run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?; if testsuite.interactive { From 8ed93743daadd66a13e2c202e87bb09fd5e365ec Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 1 Oct 2026 16:10:34 -0700 Subject: [PATCH 2/6] chore(tmachine): remove supplemental Markdown changes Signed-off-by: Matthew Grossman --- .agents/skills/helm-dev-environment/SKILL.md | 16 ------ skills/debug-openshell-cluster/SKILL.md | 9 --- tests/README.md | 58 -------------------- 3 files changed, 83 deletions(-) delete mode 100644 tests/README.md diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index be198f04c7..cb3023f476 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -296,22 +296,6 @@ KUBECONFIG=kubeconfig helm upgrade openshell deploy/helm/openshell \ Use the IP that pods in that cluster use to reach listeners on the test host. -### tmachine PostgreSQL/mTLS conformance - -For a disposable VM using staged candidate artifacts, run: - -```shell -nix run .#tmachine -- test ubuntu-k3s k3s-ha-tls conformance -``` - -This installer uses three gateway replicas on one K3s node, the pinned e2e -PostgreSQL fixture and chart-generated mTLS. It registers the guest CLI at -`https://localhost:17670` and checks an authenticated API after every boot. -The existing `k3s` installer remains the SQLite/plaintext baseline. See -`tests/README.md` for artifact prerequisites and interactive guest access. -This lane runs ordinary conformance; scale/owner-loss/rollout scenarios and -node-level HA qualification require separate coverage. - ### BackendTLSPolicy (end-to-end TLS) To enable end-to-end TLS between the Gateway proxy and the gateway pod, add diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 51ca49e714..6382428946 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -447,15 +447,6 @@ stores encrypted credential envelopes in the OpenShell database. For mentioning `server.credentialDrivers` means the values selected multiple external credential backends. -For a disposable single-node PostgreSQL/mTLS test deployment, check that the -CLI registration uses an HTTPS endpoint and that the named gateway's mTLS -directory contains the chart client certificate and CA. An open port-forward -or successful `gateway add` alone does not establish API readiness: require a -successful `openshell sandbox list --output json` as well. After the cluster -reboots, wait for PostgreSQL, the controller and all gateway replicas before -diagnosing sandbox operations. Collect Kubernetes gateway logs rather than a -host `openshell-gateway.service` journal for a Helm deployment. - For HA or PostgreSQL-backed installs, also check the external database Secret referenced by `server.externalDbSecret` and the PostgreSQL workload when it is deployed in-cluster: diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 83ddd84746..0000000000 --- a/tests/README.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# tmachine tests - -`tmachine` boots a disposable QEMU guest, prepares a runtime environment, -installs candidate artifacts, and runs a test archive. Its configuration lives -in `tests/config.nix`. The guest uses four CPUs and 4 GiB of RAM; tests run on -the host's native architecture with HVF on Apple Silicon or KVM on Linux. - -## K3s conformance - -Stage the native musl CLI, candidate gateway/sandbox/supervisor OCI archives, -Helm chart and conformance nextest bundle under `artifacts/`. The existing -`build-artifacts` flake app builds these locally; CI's -`prepare-integration-inputs.yml` stages them from the same source revision. - -The `ubuntu-k3s` environment installs K3s, Helm and nextest. Choose the basic -single-replica SQLite/plaintext installer or the PostgreSQL/mTLS installer: - -```shell -nix run .#tmachine -- test ubuntu-k3s k3s conformance -nix run .#tmachine -- test ubuntu-k3s k3s-ha-tls conformance -``` - -`k3s-ha-tls` installs three gateway replicas as a Deployment, the pinned -PostgreSQL fixture from `e2e/kubernetes/postgres-fixture.yaml`, and the chart's -built-in PKI. The gateway and peer connections use TLS; the CLI uses the chart -client certificate and `https://localhost:17670`. Unauthenticated users are -disabled. PostgreSQL uses disposable storage and test-only credentials, with -an unencrypted database connection inside the isolated guest. The fixture -image and Agent Sandbox controller are pulled by the guest during setup. - -Installer `prepare_playbooks` run before every test suite, including boots -from a cached installation. K3s preparation waits for the node, PostgreSQL -when present, controller, gateway workload and a successful sandbox-list RPC. -Client certificate files are installed with mode `0600`. A restarting systemd -port-forward exposes the gateway only on guest loopback. Readiness and -conformance failures print cluster inventory, events and bounded workload and -service logs without reading Secret contents. - -For an interactive guest using the same installer: - -```shell -nix run .#tmachine -- test ubuntu-k3s k3s-ha-tls shell -``` - -Inside the guest, use `sudo k3s kubectl` for cluster inspection. Conformance -runs as the `tmachine` user using its registered gateway and certificate -bundle. Each test boot gets a disposable overlay of the installed disk. - -The branch conformance matrix and manual Integration Test workflow include -the PostgreSQL/mTLS tuple. This setup runs the existing portable conformance -scenarios. HA fault injection and migration of #3825's scale/owner-loss/rollout -scenarios are follow-up work. Three replicas on one K3s node do not establish -node-level HA, database HA, ingress behavior or uninterrupted stream recovery. From e6e45095247feed654af6c9064d81b4979e06815 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 1 Oct 2026 16:24:49 -0700 Subject: [PATCH 3/6] refactor(tmachine): limit HA setup to installer prerequisites Signed-off-by: Matthew Grossman --- .github/workflows/branch-e2e.yml | 1 - .github/workflows/integration-runner.yml | 9 - .github/workflows/integration-test.yml | 1 - tests/ansible/playbooks/conformance/cli.yaml | 15 +- .../playbooks/openshell-k3s-ha-tls-ready.yaml | 80 ++++ .../playbooks/openshell-k3s-ready.yaml | 10 - tests/ansible/playbooks/openshell-k3s.yaml | 387 +++++++++--------- .../files/collect.sh | 50 --- .../openshell_k3s_diagnostics/tasks/main.yaml | 31 -- .../roles/openshell_k3s_ready/tasks/main.yaml | 116 ------ tests/config.nix | 3 +- 11 files changed, 270 insertions(+), 433 deletions(-) create mode 100644 tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml delete mode 100644 tests/ansible/playbooks/openshell-k3s-ready.yaml delete mode 100644 tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh delete mode 100644 tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml delete mode 100644 tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 643e5fc6e9..49cf199a3c 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -225,7 +225,6 @@ jobs: [ {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"} ] diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index c7dbd8c58a..768879c7b8 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -85,13 +85,4 @@ jobs: ENVIRONMENT: ${{ matrix.environment }} INSTALLER: ${{ matrix.installer }} TESTSUITE: ${{ matrix.testsuite }} - TMACHINE_DIAGNOSTICS_DIR: ${{ github.workspace }}/artifacts/tmachine-diagnostics run: nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}" - - - name: Upload tmachine diagnostics - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }} - path: artifacts/tmachine-diagnostics - if-no-files-found: ignore diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ec8521cc61..bc91802882 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -24,7 +24,6 @@ on: [ {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha-tls","testsuite":"conformance"}, {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"}, {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"} ] diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index 4fff19b778..fb3220b07e 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -85,17 +85,6 @@ var: conformance_result when: conformance_result.rc != 0 - - name: Detect K3s for conformance diagnostics - ansible.builtin.stat: - path: /usr/local/bin/k3s - register: conformance_k3s - when: conformance_result.rc != 0 - - - name: Collect Kubernetes conformance diagnostics - ansible.builtin.include_role: - name: openshell_k3s_diagnostics - when: conformance_result.rc != 0 and conformance_k3s.stat.exists - - name: Read OpenShell gateway logs become: true ansible.builtin.command: @@ -109,12 +98,12 @@ register: openshell_gateway_logs changed_when: false failed_when: false - when: conformance_result.rc != 0 and not conformance_k3s.stat.exists + when: conformance_result.rc != 0 - name: Show OpenShell gateway logs ansible.builtin.debug: var: openshell_gateway_logs.stdout_lines - when: conformance_result.rc != 0 and not conformance_k3s.stat.exists + when: conformance_result.rc != 0 - name: Require OpenShell conformance success ansible.builtin.assert: diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml new file mode 100644 index 0000000000..d56c4ff95b --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Prepare PostgreSQL and mTLS gateway for conformance + hosts: all + become: true + gather_facts: false + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Wait for K3s node + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --request-timeout=190s, wait, --for=condition=Ready, nodes, --all, --timeout=180s] + register: k3s_node + until: k3s_node.rc == 0 + retries: 12 + delay: 5 + changed_when: false + + - name: Wait for PostgreSQL, controller and gateway + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, "{{ item.namespace }}", rollout, status, "{{ item.workload }}", --timeout=300s] + loop: + - namespace: openshell + workload: deployment/openshell-e2e-postgres + - namespace: agent-sandbox-system + workload: deployment/agent-sandbox-controller + - namespace: openshell + workload: deployment/openshell + changed_when: false + + - name: Restart gateway forwarder after workload readiness + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + state: restarted + + - name: Read chart client TLS Secret + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] + register: k3s_client_tls + changed_when: false + no_log: true + + - name: Create guest mTLS directory + ansible.builtin.file: + path: /home/tmachine/.config/openshell/gateways/tmachine/mtls + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Write guest client TLS bundle + ansible.builtin.copy: + content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" + dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" + owner: tmachine + group: tmachine + mode: "0600" + loop: [ca.crt, tls.crt, tls.key] + no_log: true + + - name: Register guest gateway + ansible.builtin.include_role: + name: openshell_client + apply: + become: false + vars: + openshell_client_gateway_endpoint: https://localhost:17670 + + - name: Wait for authenticated gateway API + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, sandbox, list, --output, json] + register: k3s_gateway_api + until: k3s_gateway_api.rc == 0 + retries: 24 + delay: 5 + changed_when: false diff --git a/tests/ansible/playbooks/openshell-k3s-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ready.yaml deleted file mode 100644 index 75fc711dcc..0000000000 --- a/tests/ansible/playbooks/openshell-k3s-ready.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Prepare K3s gateway after every test boot - hosts: all - become: true - gather_facts: false - roles: - - openshell_k3s_ready diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index 14bf5e4fd2..130b08de7c 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -9,216 +9,203 @@ vars: k3s_ha: "{{ openshell_k3s_ha | default(false) | bool }}" tasks: - - name: Install K3s gateway and fixtures + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Install OpenShell CLI + ansible.builtin.copy: + src: "{{ openshell_cli_binary }}" + dest: /usr/local/bin/openshell + mode: "0755" + + - name: Create OpenShell artifact directory + ansible.builtin.file: + path: /var/lib/openshell/artifacts + state: directory + mode: "0700" + + - name: Copy OpenShell images + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" + mode: "0600" + loop: + - name: gateway + src: "{{ openshell_gateway_image }}" + - name: sandbox + src: "{{ openshell_sandbox_image }}" + - name: supervisor + src: "{{ openshell_supervisor_image }}" + + - name: Import OpenShell images into K3s + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - ctr + - --namespace + - k8s.io + - images + - import + - "/var/lib/openshell/artifacts/{{ item }}.tar" + loop: + - gateway + - sandbox + - supervisor + + - name: Copy OpenShell Helm chart + ansible.builtin.copy: + src: "{{ openshell_helm_chart }}" + dest: /var/lib/openshell/artifacts/helm-chart.tgz + mode: "0600" + + - name: Write OpenShell Helm values + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values.yaml + mode: "0600" + content: | + global: + image: + registry: "" + {% if k3s_ha %} + replicaCount: 3 + workload: + kind: deployment + {% endif %} + gateway: + image: + repository: openshell/gateway + tag: tmachine + pullPolicy: Never + sandboxRuntime: + image: + repository: openshell/sandbox + tag: tmachine + pullPolicy: never + supervisor: + image: + repository: openshell/supervisor + tag: tmachine + pullPolicy: never + networkPolicy: + enabled: true + server: + auth: + allowUnauthenticatedUsers: {{ (not k3s_ha) | to_json }} + disableTls: {{ (not k3s_ha) | to_json }} + telemetryEnabled: false + {% if k3s_ha %} + externalDbSecret: openshell-e2e-postgres-credentials + {% endif %} + + - name: Install PostgreSQL fixture + when: k3s_ha block: - - name: Wait for SSH - ansible.builtin.wait_for_connection: - - - name: Install OpenShell CLI - ansible.builtin.copy: - src: "{{ openshell_cli_binary }}" - dest: /usr/local/bin/openshell - mode: "0755" - - - name: Create OpenShell artifact directory - ansible.builtin.file: - path: /var/lib/openshell/artifacts - state: directory - mode: "0700" - - - name: Copy OpenShell images - ansible.builtin.copy: - src: "{{ item.src }}" - dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" - mode: "0600" - loop: - - name: gateway - src: "{{ openshell_gateway_image }}" - - name: sandbox - src: "{{ openshell_sandbox_image }}" - - name: supervisor - src: "{{ openshell_supervisor_image }}" - - - name: Import OpenShell images into K3s + - name: Create OpenShell namespace ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - ctr - - --namespace - - k8s.io - - images - - import - - "/var/lib/openshell/artifacts/{{ item }}.tar" - loop: - - gateway - - sandbox - - supervisor - - - name: Copy OpenShell Helm chart - ansible.builtin.copy: - src: "{{ openshell_helm_chart }}" - dest: /var/lib/openshell/artifacts/helm-chart.tgz - mode: "0600" + argv: [/usr/local/bin/k3s, kubectl, create, namespace, openshell] - - name: Write OpenShell Helm values + - name: Copy PostgreSQL fixture ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values.yaml + src: "{{ openshell_postgres_fixture }}" + dest: /var/lib/openshell/artifacts/postgres.yaml mode: "0600" - content: | - global: - image: - registry: "" - replicaCount: {{ 3 if k3s_ha else 1 }} - workload: - kind: {{ 'deployment' if k3s_ha else 'statefulset' }} - gateway: - image: - repository: openshell/gateway - tag: tmachine - pullPolicy: Never - sandboxRuntime: - image: - repository: openshell/sandbox - tag: tmachine - pullPolicy: never - supervisor: - image: - repository: openshell/supervisor - tag: tmachine - pullPolicy: never - networkPolicy: - enabled: true - server: - auth: - allowUnauthenticatedUsers: {{ (not k3s_ha) | to_json }} - disableTls: {{ (not k3s_ha) | to_json }} - telemetryEnabled: false - {% if k3s_ha %} - externalDbSecret: openshell-e2e-postgres-credentials - peer: - allowInsecureTransport: false - credentialDrivers: - kubernetesSecrets: - enabled: true - namespace: openshell - {% endif %} - - - name: Install PostgreSQL fixture - when: k3s_ha - block: - - name: Create OpenShell namespace - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, create, namespace, openshell] - - - name: Copy pinned PostgreSQL fixture - ansible.builtin.copy: - src: "{{ openshell_postgres_fixture }}" - dest: /var/lib/openshell/artifacts/postgres.yaml - mode: "0600" - - - name: Apply PostgreSQL fixture - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] - - - name: Wait for PostgreSQL - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] - changed_when: false - - - name: Install Agent Sandbox - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - apply - - --filename - - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" - - - name: Wait for Agent Sandbox CRD - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - wait - - --for=condition=Established - - crd/sandboxes.agents.x-k8s.io - - --timeout=120s - changed_when: false - - name: Wait for Agent Sandbox controller + - name: Apply PostgreSQL fixture ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - --namespace - - agent-sandbox-system - - rollout - - status - - deployment/agent-sandbox-controller - - --timeout=300s - changed_when: false + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] - - name: Install OpenShell Helm chart + - name: Wait for PostgreSQL ansible.builtin.command: - argv: - - /usr/local/bin/helm - - install - - openshell - - /var/lib/openshell/artifacts/helm-chart.tgz - - --namespace - - openshell - - --create-namespace - - --values - - /var/lib/openshell/artifacts/values.yaml - - --wait - - --timeout=5m - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - - name: Install gateway port-forward service - ansible.builtin.copy: - dest: /etc/systemd/system/openshell-k3s-port-forward.service - mode: "0644" - content: | - [Unit] - Description=OpenShell K3s gateway port forward - After=k3s.service - Requires=k3s.service - StartLimitIntervalSec=0 - - [Service] - ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 - Restart=always - RestartSec=5 - - [Install] - WantedBy=multi-user.target - - - name: Start gateway port-forward service - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - daemon_reload: true - enabled: true - state: started - - - name: Wait for OpenShell gateway - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 17670 - timeout: 60 - - rescue: - - name: Collect installation diagnostics - ansible.builtin.include_role: - name: openshell_k3s_diagnostics - - - name: Fail K3s installation - ansible.builtin.fail: - msg: K3s gateway installation failed; see the cluster diagnostics above. + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] + changed_when: false -- name: Prepare OpenShell gateway for test client + - name: Install Agent Sandbox + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - apply + - --filename + - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" + + - name: Wait for Agent Sandbox CRD + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - wait + - --for=condition=Established + - crd/sandboxes.agents.x-k8s.io + - --timeout=120s + changed_when: false + + - name: Wait for Agent Sandbox controller + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - agent-sandbox-system + - rollout + - status + - deployment/agent-sandbox-controller + - --timeout=300s + changed_when: false + + - name: Install OpenShell Helm chart + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - install + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --create-namespace + - --values + - /var/lib/openshell/artifacts/values.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + + - name: Install gateway port-forward service + ansible.builtin.copy: + dest: /etc/systemd/system/openshell-k3s-port-forward.service + mode: "0644" + content: | + [Unit] + Description=OpenShell K3s gateway port forward + After=k3s.service + Requires=k3s.service + + [Service] + ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 + Restart=always + RestartSec=1 + + [Install] + WantedBy=multi-user.target + + - name: Start gateway port-forward service + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + daemon_reload: true + enabled: true + state: started + + - name: Wait for OpenShell gateway + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 17670 + timeout: 60 + +- name: Register OpenShell gateway for test client hosts: all - become: true gather_facts: false roles: - - openshell_k3s_ready + - role: openshell_client + when: not (openshell_k3s_ha | default(false) | bool) + +- import_playbook: openshell-k3s-ha-tls-ready.yaml + when: openshell_k3s_ha | default(false) | bool diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh deleted file mode 100644 index 1f61c736be..0000000000 --- a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# No kubeconfig, Secrets, environment dumps, or complete Pod specifications. -set -u -umask 077 -mkdir -p /var/lib/openshell/diagnostics -output=/var/lib/openshell/diagnostics/k3s.txt - -collect() { - printf '\n### %s\n' "$1" - shift - # Bound both stalled commands and unusually large log streams. - timeout 10s "$@" 2>&1 | tail -c 262144 - printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" -} - -{ - date --utc --iso-8601=seconds - collect 'Service state' systemctl show k3s.service openshell-k3s-port-forward.service \ - -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \ - -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ - -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec - collect 'Forwarder unit' systemctl cat openshell-k3s-port-forward.service - collect 'K3s and forwarder boot journals' journalctl -b --no-pager --lines=500 \ - -u k3s.service -u openshell-k3s-port-forward.service - collect 'Gateway listener' ss -ltnp 'sport = :17670' - collect 'K3s version' /usr/local/bin/k3s --version - collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz - collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide - collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ - -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' - collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \ - -o 'custom-columns=NAME:.metadata.name,SELECTOR:.spec.selector,PORTS:.spec.ports' - collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ - -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' - collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp - collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs -l app.kubernetes.io/name=openshell -c openshell-gateway --max-log-requests=5 --tail=300 --timestamps - collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs -l app.kubernetes.io/name=openshell -c openshell-gateway --max-log-requests=5 --previous --tail=300 --timestamps - collect 'PostgreSQL logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs deployment/openshell-e2e-postgres --tail=100 - collect 'Controller logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs deployment/agent-sandbox-controller --tail=100 - collect 'Memory' free -m - collect 'Disk' df -h / /var/lib/rancher/k3s -} | sed -E \ - -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \ - -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' \ - -e 's#(postgres(ql)?://)[^/@ ]+@#\1[REDACTED]@#gI' \ - > "$output" -cat "$output" diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml deleted file mode 100644 index e1ee0ba622..0000000000 --- a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Check for K3s - become: true - ansible.builtin.stat: - path: /usr/local/bin/k3s - register: k3s_diagnostics_binary - failed_when: false - -- name: Preserve K3s diagnostics before the VM exits - when: k3s_diagnostics_binary.stat.exists | default(false) - become: true - block: - - name: Collect bounded K3s diagnostics - ansible.builtin.script: collect.sh - register: k3s_diagnostics_collection - changed_when: false - failed_when: false - - - name: Show K3s diagnostics - ansible.builtin.debug: - var: k3s_diagnostics_collection.stdout_lines - - - name: Fetch K3s diagnostics - ansible.builtin.fetch: - src: /var/lib/openshell/diagnostics/k3s.txt - dest: "{{ lookup('env', 'TMACHINE_DIAGNOSTICS_DIR') | default(role_path + '/../../../../artifacts/tmachine-diagnostics', true) }}/{{ inventory_hostname }}/k3s.txt" - flat: true - failed_when: false diff --git a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml deleted file mode 100644 index 05ac879068..0000000000 --- a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Wait for SSH - ansible.builtin.wait_for_connection: - -- name: Resolve K3s gateway configuration - ansible.builtin.set_fact: - k3s_ha: "{{ openshell_k3s_ha | default(false) | bool }}" - -- name: Prepare K3s gateway - block: - - name: Wait for K3s API - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --request-timeout=10s, get, nodes, --output, name] - register: k3s_api - until: k3s_api.rc == 0 - retries: 30 - delay: 5 - changed_when: false - - - name: Wait for K3s node - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --request-timeout=190s, wait, --for=condition=Ready, nodes, --all, --timeout=180s] - changed_when: false - - - name: Wait for PostgreSQL after boot - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] - changed_when: false - when: k3s_ha - - - name: Wait for Agent Sandbox controller - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, agent-sandbox-system, rollout, status, deployment/agent-sandbox-controller, --timeout=300s] - changed_when: false - - - name: Wait for gateway workload - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, "{{ 'deployment' if k3s_ha else 'statefulset' }}/openshell", --timeout=300s] - changed_when: false - - - name: Start gateway port-forward service - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - state: started - - - name: Wait for gateway listener - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 17670 - timeout: 120 - - - name: Install mTLS client identity - when: k3s_ha - block: - - name: Read chart client TLS Secret - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] - register: k3s_client_tls - changed_when: false - no_log: true - - - name: Create guest client config directories - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: tmachine - group: tmachine - mode: "0700" - loop: - - /home/tmachine/.config - - /home/tmachine/.config/openshell - - /home/tmachine/.config/openshell/gateways - - /home/tmachine/.config/openshell/gateways/tmachine - - /home/tmachine/.config/openshell/gateways/tmachine/mtls - - - name: Write guest client TLS bundle - ansible.builtin.copy: - content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" - dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" - owner: tmachine - group: tmachine - mode: "0600" - loop: [ca.crt, tls.crt, tls.key] - no_log: true - - - name: Register guest gateway - ansible.builtin.include_role: - name: openshell_client - apply: - become: false - vars: - openshell_client_gateway_endpoint: "{{ 'https://localhost:17670' if k3s_ha else 'http://127.0.0.1:17670' }}" - - # gateway add can warn about an unreachable endpoint and still return zero. - # A sandbox-list RPC exercises TLS, authentication and the shared database. - - name: Wait for authenticated gateway API - become: false - ansible.builtin.command: - argv: [/usr/local/bin/openshell, sandbox, list, --output, json] - register: k3s_gateway_api - until: k3s_gateway_api.rc == 0 - retries: 24 - delay: 5 - changed_when: false - - rescue: - - name: Collect K3s diagnostics - ansible.builtin.include_role: - name: openshell_k3s_diagnostics - - - name: Fail K3s preparation - ansible.builtin.fail: - msg: K3s gateway preparation failed; see the cluster diagnostics above. diff --git a/tests/config.nix b/tests/config.nix index d8a0a6648d..fe5af9b3b7 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -85,7 +85,6 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; - prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; inputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; @@ -99,7 +98,7 @@ let name = "k3s-ha-tls"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; - prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; + prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls-ready.yaml" ]; inputs = { openshell_k3s_ha = "true"; openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; From f870a4b98ec82b3be003ce64a58326bbb85dba4a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 1 Oct 2026 16:33:58 -0700 Subject: [PATCH 4/6] refactor(tmachine): compose HA TLS setup from the K3s installer Signed-off-by: Matthew Grossman --- .../playbooks/openshell-k3s-ha-tls-ready.yaml | 5 ++ .../playbooks/openshell-k3s-ha-tls.yaml | 59 +++++++++++++++++++ tests/ansible/playbooks/openshell-k3s.yaml | 42 +------------ tests/config.nix | 3 +- 4 files changed, 68 insertions(+), 41 deletions(-) create mode 100644 tests/ansible/playbooks/openshell-k3s-ha-tls.yaml diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml index d56c4ff95b..db67b1a1d2 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml @@ -61,6 +61,11 @@ loop: [ca.crt, tls.crt, tls.key] no_log: true + - name: Remove previous guest gateway registration + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, gateway, remove, tmachine] + - name: Register guest gateway ansible.builtin.include_role: name: openshell_client diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml new file mode 100644 index 0000000000..a5b3c09a32 --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- import_playbook: openshell-k3s.yaml + +- name: Add PostgreSQL and TLS to the K3s test installation + hosts: all + become: true + gather_facts: false + tasks: + - name: Copy PostgreSQL fixture + ansible.builtin.copy: + src: "{{ openshell_postgres_fixture }}" + dest: /var/lib/openshell/artifacts/postgres.yaml + mode: "0600" + + - name: Apply PostgreSQL fixture + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] + + - name: Wait for PostgreSQL + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] + changed_when: false + + - name: Write PostgreSQL and TLS values overlay + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values-ha-tls.yaml + mode: "0600" + content: | + replicaCount: 3 + workload: + kind: deployment + server: + externalDbSecret: openshell-e2e-postgres-credentials + disableTls: false + auth: + allowUnauthenticatedUsers: false + + - name: Apply PostgreSQL and TLS overlay to OpenShell + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - upgrade + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --values + - /var/lib/openshell/artifacts/values.yaml + - --values + - /var/lib/openshell/artifacts/values-ha-tls.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + +- import_playbook: openshell-k3s-ha-tls-ready.yaml diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index 130b08de7c..dd00aa4bda 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -6,8 +6,6 @@ hosts: all become: true gather_facts: false - vars: - k3s_ha: "{{ openshell_k3s_ha | default(false) | bool }}" tasks: - name: Wait for SSH ansible.builtin.wait_for_connection: @@ -66,11 +64,6 @@ global: image: registry: "" - {% if k3s_ha %} - replicaCount: 3 - workload: - kind: deployment - {% endif %} gateway: image: repository: openshell/gateway @@ -90,34 +83,9 @@ enabled: true server: auth: - allowUnauthenticatedUsers: {{ (not k3s_ha) | to_json }} - disableTls: {{ (not k3s_ha) | to_json }} + allowUnauthenticatedUsers: true + disableTls: true telemetryEnabled: false - {% if k3s_ha %} - externalDbSecret: openshell-e2e-postgres-credentials - {% endif %} - - - name: Install PostgreSQL fixture - when: k3s_ha - block: - - name: Create OpenShell namespace - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, create, namespace, openshell] - - - name: Copy PostgreSQL fixture - ansible.builtin.copy: - src: "{{ openshell_postgres_fixture }}" - dest: /var/lib/openshell/artifacts/postgres.yaml - mode: "0600" - - - name: Apply PostgreSQL fixture - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, apply, --filename, /var/lib/openshell/artifacts/postgres.yaml] - - - name: Wait for PostgreSQL - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, rollout, status, deployment/openshell-e2e-postgres, --timeout=300s] - changed_when: false - name: Install Agent Sandbox ansible.builtin.command: @@ -204,8 +172,4 @@ hosts: all gather_facts: false roles: - - role: openshell_client - when: not (openshell_k3s_ha | default(false) | bool) - -- import_playbook: openshell-k3s-ha-tls-ready.yaml - when: openshell_k3s_ha | default(false) | bool + - openshell_client diff --git a/tests/config.nix b/tests/config.nix index fe5af9b3b7..bd175aeb13 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -97,10 +97,9 @@ let { name = "k3s-ha-tls"; use_galaxy = false; - playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; + playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls.yaml" ]; prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls-ready.yaml" ]; inputs = { - openshell_k3s_ha = "true"; openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; From 74ab8ffd07fa3871cc5bcc8f3c679548af717eeb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 1 Oct 2026 17:04:32 -0700 Subject: [PATCH 5/6] fix(tmachine): admit certificate-verified users in K3s fixture Signed-off-by: Matthew Grossman --- tests/ansible/playbooks/openshell-k3s-ha-tls.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml index a5b3c09a32..ca9086a408 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml @@ -36,7 +36,9 @@ externalDbSecret: openshell-e2e-postgres-credentials disableTls: false auth: - allowUnauthenticatedUsers: false + # Kubernetes has no mTLS user-identity mode. Require the client + # certificate at the TLS layer and use the fixture's dev user. + allowUnauthenticatedUsers: true - name: Apply PostgreSQL and TLS overlay to OpenShell ansible.builtin.command: From b6bd1508f315804f03952144e262071f111508c8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 1 Oct 2026 17:53:30 -0700 Subject: [PATCH 6/6] refactor(tmachine): keep HA fixture setup in the installer Signed-off-by: Matthew Grossman --- .../playbooks/openshell-k3s-ha-tls-ready.yaml | 85 ------------------- .../playbooks/openshell-k3s-ha-tls.yaml | 53 +++++++++++- tests/config.nix | 1 - tests/tmachine/src/config.rs | 28 ------ tests/tmachine/src/qemu/test.rs | 1 - 5 files changed, 52 insertions(+), 116 deletions(-) delete mode 100644 tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml deleted file mode 100644 index db67b1a1d2..0000000000 --- a/tests/ansible/playbooks/openshell-k3s-ha-tls-ready.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Prepare PostgreSQL and mTLS gateway for conformance - hosts: all - become: true - gather_facts: false - tasks: - - name: Wait for SSH - ansible.builtin.wait_for_connection: - - - name: Wait for K3s node - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --request-timeout=190s, wait, --for=condition=Ready, nodes, --all, --timeout=180s] - register: k3s_node - until: k3s_node.rc == 0 - retries: 12 - delay: 5 - changed_when: false - - - name: Wait for PostgreSQL, controller and gateway - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, "{{ item.namespace }}", rollout, status, "{{ item.workload }}", --timeout=300s] - loop: - - namespace: openshell - workload: deployment/openshell-e2e-postgres - - namespace: agent-sandbox-system - workload: deployment/agent-sandbox-controller - - namespace: openshell - workload: deployment/openshell - changed_when: false - - - name: Restart gateway forwarder after workload readiness - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - state: restarted - - - name: Read chart client TLS Secret - ansible.builtin.command: - argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] - register: k3s_client_tls - changed_when: false - no_log: true - - - name: Create guest mTLS directory - ansible.builtin.file: - path: /home/tmachine/.config/openshell/gateways/tmachine/mtls - state: directory - owner: tmachine - group: tmachine - mode: "0700" - - - name: Write guest client TLS bundle - ansible.builtin.copy: - content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" - dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" - owner: tmachine - group: tmachine - mode: "0600" - loop: [ca.crt, tls.crt, tls.key] - no_log: true - - - name: Remove previous guest gateway registration - become: false - ansible.builtin.command: - argv: [/usr/local/bin/openshell, gateway, remove, tmachine] - - - name: Register guest gateway - ansible.builtin.include_role: - name: openshell_client - apply: - become: false - vars: - openshell_client_gateway_endpoint: https://localhost:17670 - - - name: Wait for authenticated gateway API - become: false - ansible.builtin.command: - argv: [/usr/local/bin/openshell, sandbox, list, --output, json] - register: k3s_gateway_api - until: k3s_gateway_api.rc == 0 - retries: 24 - delay: 5 - changed_when: false diff --git a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml index ca9086a408..51df48660c 100644 --- a/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml +++ b/tests/ansible/playbooks/openshell-k3s-ha-tls.yaml @@ -58,4 +58,55 @@ environment: KUBECONFIG: /etc/rancher/k3s/k3s.yaml -- import_playbook: openshell-k3s-ha-tls-ready.yaml + - name: Restart gateway forwarder after TLS upgrade + ansible.builtin.systemd_service: + name: openshell-k3s-port-forward.service + state: restarted + + - name: Read chart client TLS Secret + ansible.builtin.command: + argv: [/usr/local/bin/k3s, kubectl, --namespace, openshell, get, secret, openshell-client-tls, --output, json] + register: k3s_client_tls + changed_when: false + no_log: true + + - name: Create guest mTLS directory + ansible.builtin.file: + path: /home/tmachine/.config/openshell/gateways/tmachine/mtls + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Write guest client TLS bundle + ansible.builtin.copy: + content: "{{ (k3s_client_tls.stdout | from_json).data[item] | b64decode }}" + dest: "/home/tmachine/.config/openshell/gateways/tmachine/mtls/{{ item }}" + owner: tmachine + group: tmachine + mode: "0600" + loop: [ca.crt, tls.crt, tls.key] + no_log: true + + - name: Remove baseline guest gateway registration + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, gateway, remove, tmachine] + + - name: Register guest gateway + ansible.builtin.include_role: + name: openshell_client + apply: + become: false + vars: + openshell_client_gateway_endpoint: https://localhost:17670 + + - name: Wait for authenticated gateway API + become: false + ansible.builtin.command: + argv: [/usr/local/bin/openshell, sandbox, list, --output, json] + register: k3s_gateway_api + until: k3s_gateway_api.rc == 0 + retries: 24 + delay: 5 + changed_when: false diff --git a/tests/config.nix b/tests/config.nix index bd175aeb13..2cb6d99a04 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -98,7 +98,6 @@ let name = "k3s-ha-tls"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls.yaml" ]; - prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ha-tls-ready.yaml" ]; inputs = { openshell_postgres_fixture = "../e2e/kubernetes/postgres-fixture.yaml"; agent_sandbox_version = "0.5.0"; diff --git a/tests/tmachine/src/config.rs b/tests/tmachine/src/config.rs index 6e6abfd009..bc19cbbb1f 100644 --- a/tests/tmachine/src/config.rs +++ b/tests/tmachine/src/config.rs @@ -40,34 +40,6 @@ pub struct Installer { pub use_galaxy: bool, pub playbooks: Vec, pub inputs: BTreeMap, - /// Run on every test boot, including when the installed disk is cached. - #[serde(default)] - pub prepare_playbooks: Vec, -} - -#[cfg(test)] -mod tests { - use super::Installer; - - #[test] - fn existing_installers_need_no_boot_preparation() { - let installer: Installer = - serde_saphyr::from_str("name: none\nuse_galaxy: false\nplaybooks: []\ninputs: {}\n") - .unwrap(); - assert!(installer.prepare_playbooks.is_empty()); - } - - #[test] - fn installer_can_request_boot_preparation() { - let installer: Installer = serde_saphyr::from_str( - "name: k3s\nuse_galaxy: false\nplaybooks: []\ninputs: {}\nprepare_playbooks: [ready.yaml]\n", - ) - .unwrap(); - assert_eq!( - installer.prepare_playbooks, - [std::path::PathBuf::from("ready.yaml")] - ); - } } #[derive(Clone, Deserialize)] diff --git a/tests/tmachine/src/qemu/test.rs b/tests/tmachine/src/qemu/test.rs index 0da66fe088..ac8a8bdfc0 100644 --- a/tests/tmachine/src/qemu/test.rs +++ b/tests/tmachine/src/qemu/test.rs @@ -26,7 +26,6 @@ pub async fn test( let image = QemuImage::create(&install_disk, test_disk).await; let vm = QemuVm::start(&image).await; - run_playbooks(&installer.prepare_playbooks, &installer.inputs).await?; run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?; if testsuite.interactive {