diff --git a/docs/development/testing.md b/docs/development/testing.md index 600058df..31cf8bc8 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -75,6 +75,8 @@ mise run cleanup-cluster `mise run e2e-test` includes the IPv6 backend test slice. It is intended for the CAPL-provisioned clusters; do not use it against an existing LKE cluster. +`mise run e2e-test-subnet` creates a second workload cluster in the same VPC. The scenario checks that every inspected node's VPC interface contains its assigned Pod CIDR and no Pod CIDRs assigned to nodes in the other cluster. It polls missing routes every five seconds, with one three-minute convergence deadline shared across both clusters. Foreign routes and Kubernetes or Linode API errors fail the test rather than being retried. Pass/fail is based on interface state, not IP-like text in CCM logs. + The `lb-with-reserved-backend-ipv4-range` scenario is the one exception to the "a test only touches its own namespace" rule. It adds `--nodebalancer-backend-ipv4-subnet` and `--nodebalancer-backend-ipv4-reserved-range` to the `ccm-linode` DaemonSet, waits for the rollout, and removes both flags again when the test finishes. It is marked `concurrent: false` so Chainsaw runs it on its own, and it carries no `lke` label because it expects VPC-backed NodeBalancers and a DaemonSet it is allowed to edit. If the run is interrupted between the patch and the cleanup, remove the two flags by hand before running other scenarios. ## End-to-End Tests with LKE diff --git a/e2e/subnet-test/chainsaw-test.yaml b/e2e/subnet-test/chainsaw-test.yaml index c56ba042..aee651d0 100644 --- a/e2e/subnet-test/chainsaw-test.yaml +++ b/e2e/subnet-test/chainsaw-test.yaml @@ -20,7 +20,7 @@ spec: echo "Test failed. Fetching CCM logs..." kubectl logs -n kube-system daemonsets/ccm-linode | grep "subnet-filtering-test" | tail -100 steps: - - name: Check if the CCM for each cluster focus on their individual subnets + - name: Wait for each cluster's routes and check subnet isolation try: - script: shell: bash @@ -34,57 +34,61 @@ spec: exit 1 fi - # Iterate through both clusters - for config in "$FIRST_CONFIG" "$SECOND_CONFIG"; do - # Get all node names - nodes=$(KUBECONFIG=$config kubectl get nodes -o jsonpath='{.items[*].metadata.name}') - if [ -z "$nodes" ]; then - echo "Error: No nodes found in cluster" - exit 1 - fi - - # Process each node - for node in $nodes; do - echo "Checking node: $node" - - # Get pod CIDR and instance ID - pod_cidr=$(KUBECONFIG=$config kubectl get node "$node" -o jsonpath='{.spec.podCIDR}') - instance_id=$(KUBECONFIG=$config kubectl get node "$node" -o jsonpath='{.spec.providerID}' | sed 's/linode:\/\///') - - echo " Pod CIDR: $pod_cidr" - echo " Instance ID: $instance_id" - - # Get interface details for this config - interfaces=$(curl -s \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "$LINODE_URL/v4/linode/instances/$instance_id/configs" \ - | jq -r '.data[0].interfaces') - - # Check if pod CIDR is in the VPC interface IP ranges - if echo "$interfaces" | jq -e --arg cidr "$pod_cidr" '.[] | select(.purpose == "vpc") | .ip_ranges[] | select(. == $cidr)' > /dev/null; then - echo "Pod CIDR found in VPC interface configuration" - else - echo "Pod CIDR not found in VPC interface configuration" - echo "Current VPC interface configuration:" - echo "$interfaces" | jq '.[] | select(.purpose == "vpc")' - fi - - echo "---" - done + first_nodes=$(KUBECONFIG="$FIRST_CONFIG" kubectl get nodes -o json --request-timeout=15s) + second_nodes=$(KUBECONFIG="$SECOND_CONFIG" kubectl get nodes -o json --request-timeout=15s) + for nodes in "$first_nodes" "$second_nodes"; do + jq -e '(.items | length > 0) and all(.items[]; + (.spec.podCIDR // "") != "" and (.spec.providerID // "" | test("^linode://[0-9]+$")))' \ + <<< "$nodes" > /dev/null done - # Grep logs of each cluster for IPs from the other cluster - echo "Checking logs of each CCM" - if ! [ $(KUBECONFIG=$FIRST_CONFIG kubectl logs daemonset/ccm-linode -n kube-system | grep "172.16" | wc -l) -eq 0 ]; then - echo "IP address from testing subnet found in logs of test cluster" - exit 1 - fi - - if ! [ $(KUBECONFIG=$SECOND_CONFIG kubectl logs daemonset/ccm-linode -n kube-system | grep "10.192" | wc -l) -eq 0 ]; then - echo "IP address from default subnet found in logs of second cluster" - exit 1 - fi + # Pair each cluster's nodes with the other cluster's pod CIDRs. + # Each tab-separated row contains node, pod CIDR, provider ID, and foreign CIDRs as JSON. + targets=$(jq -nr --argjson first "$first_nodes" --argjson second "$second_nodes" ' + [[$first, $second], [$second, $first]][] | + (.[1].items | map(.spec.podCIDR) | tojson) as $foreign | + .[0].items[] | [.metadata.name, .spec.podCIDR, .spec.providerID, $foreign] | @tsv + ') + # Node Ready does not guarantee that CCM has populated interface routes. + # Share one deadline across all nodes, within the suite's 5-minute exec timeout. + # SECONDS is Bash's built-in elapsed timer; it advances automatically. + deadline=$((SECONDS + 180)) + while true; do + observations=() + while IFS=$'\t' read -r node pod_cidr provider_id foreign_cidrs; do + # Stop starting API requests after the deadline, even partway through a node scan. + (( SECONDS < deadline )) || exit 1 + state=$(curl --fail --silent --show-error --max-time 15 \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + "$LINODE_URL/v4/linode/instances/${provider_id#linode://}/configs" \ + | jq -c --arg node "$node" --arg cidr "$pod_cidr" \ + --argjson foreign "$foreign_cidrs" ' + [.data[0].interfaces[] | select(.purpose == "vpc") | .ip_ranges[]?] as $ranges | + { + node: $node, + own_route_present: ($ranges | index($cidr) != null), + foreign_routes: [$ranges[] | select(. as $range | $foreign | index($range) != null)] + } + ') + # Never retry an isolation violation. + if jq -e '.foreign_routes | length > 0' <<< "$state" > /dev/null; then + # Return JSON successfully so Chainsaw's foreign-route assertion reports the failure. + printf '[%s]\n' "$state" + exit 0 + fi + observations+=("$state") + done <<< "$targets" + # Combine node observations into the JSON array consumed by Chainsaw below. + result=$(printf '%s\n' "${observations[@]}" | jq -s '.') + if jq -e 'all(.[]; .own_route_present)' <<< "$result" > /dev/null; then + break + fi + sleep 5 + (( SECONDS < deadline )) || break + done + printf '%s\n' "$result" check: ($error == null): true - (contains($stdout, 'Pod CIDR not found in VPC interface configuration')): false + (length(json_parse($stdout)[?own_route_present == `false`])): 0 + (length(json_parse($stdout)[].foreign_routes[])): 0