Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ jobs:
# up, and that no address reaches the public log.
- 'scripts/probe-gateway-denied-peer.sh'
- 'scripts/tests/test-probe-gateway-denied-peer.sh'
- 'scripts/tests/gateway-denied-peer-wire/**'
- '.github/workflows/probe-gateway-denied-peer.yaml'
# The #3144 admission probe, its test and its workflow are one
# contract: the test pins that every write the probe sends is a
Expand Down Expand Up @@ -2501,7 +2502,8 @@ jobs:
- name: 🚧 Validate Gateway denied-peer probe
if: needs.changes.outputs.k8s == 'true'
# The probe needs prod, runs only on dispatch and writes while it runs. Its
# test fakes kubectl and pins each verdict against a one-fixture control.
# test pins verdicts and cleanup, then executes both rendered transport modes with
# real curl and synthetic loopback HTTP/TLS peers.
run: |
shellcheck scripts/probe-gateway-denied-peer.sh scripts/tests/test-probe-gateway-denied-peer.sh
bash scripts/tests/test-probe-gateway-denied-peer.sh
Expand Down
16 changes: 13 additions & 3 deletions .github/workflows/probe-gateway-denied-peer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ jobs:

# The allowed half: the fence is in place and the load balancer's traffic is still answered.
- name: Verify the selected fence and the public routes
id: binding
shell: bash
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
Expand All @@ -84,19 +85,28 @@ jobs:
verified="$(go run ./scripts/gateway-receiver-binding verify --context admin@prod)"
printf '%s\n' "${verified}"
# Without a selected fence, an answered connection would say nothing about the fence.
grep -Eq '(^| )fence_selected=true( |$)' <<<"${verified}" || { printf 'The fence is not selected; nothing was probed.\n' >&2; exit 1; }
pattern='^GATEWAY_RECEIVER_SOURCE=VERIFIED public_probes=4 fence_selected=true proxy_protocol=(true|false)$'
[[ "${verified}" =~ ${pattern} ]] || { printf 'The fenced transport profile is unverified; nothing was probed.\n' >&2; exit 1; }
transport=plain
[[ "${BASH_REMATCH[1]}" == false ]] || transport=proxy-v1
printf 'transport=%s\n' "${transport}" >>"${GITHUB_OUTPUT}"

- name: Probe with a pod as the peer
id: probe
shell: bash
env:
RUN_ID: ${{ github.run_id }}
run: ./scripts/probe-gateway-denied-peer.sh --context admin@prod --run-id "${RUN_ID}"
TRANSPORT: ${{ steps.binding.outputs.transport }}
run: ./scripts/probe-gateway-denied-peer.sh --context admin@prod --run-id "${RUN_ID}" --transport "${TRANSPORT}"

# A cancelled job can end the step before the script's own cleanup runs.
- name: Remove anything this run left behind
if: ${{ always() && steps.probe.outcome != 'skipped' }}
shell: bash
env:
RUN_ID: ${{ github.run_id }}
run: kubectl --context admin@prod --request-timeout=30s --namespace whoami delete pods,ciliumnetworkpolicies --selector "platform.devantler.tech/gateway-peer-probe=${RUN_ID}" --ignore-not-found >/dev/null 2>&1
run: |
set -euo pipefail
kubectl --context admin@prod --request-timeout=30s --namespace whoami delete pods,ciliumnetworkpolicies --selector "platform.devantler.tech/gateway-peer-probe=${RUN_ID}" --ignore-not-found --wait=true --timeout=25s >/dev/null 2>&1
remaining="$(kubectl --context admin@prod --request-timeout=30s --namespace whoami get pods,ciliumnetworkpolicies --selector "platform.devantler.tech/gateway-peer-probe=${RUN_ID}" -o name 2>/dev/null)"
[[ -z "${remaining}" ]] || { printf 'Run-owned cleanup is unverified.\n' >&2; exit 1; }
2 changes: 1 addition & 1 deletion scripts/gateway-receiver-binding/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func main() {
fmt.Fprintf(os.Stderr, "GATEWAY_RECEIVER_SOURCE=UNVERIFIED stage=%s\n", err.Error())
os.Exit(2)
}
fmt.Printf("GATEWAY_RECEIVER_SOURCE=VERIFIED public_probes=4 fence_selected=%t\n", profile.Fence)
fmt.Printf("GATEWAY_RECEIVER_SOURCE=VERIFIED public_probes=4 fence_selected=%t proxy_protocol=%t\n", profile.Fence, profile.Proxy)
return
}
if err := prepare(ctx, kubeContext, outputPath); err != nil {
Expand Down
29 changes: 23 additions & 6 deletions scripts/probe-gateway-denied-peer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@
#
# THE QUESTION (devantler-tech/platform#5448, stage 3). Every TLS chain on the Gateway's listener
# accepts only the load balancer's private address. So far only allowed traffic has been observed,
# which cannot show the fence refuses anything. This must be answered before any PROXY change,
# which cannot show the fence refuses anything. Both listener transport modes must be testable,
# because PROXY metadata names a client the listener must not take from an unbound peer.
#
# HOW. One short-lived pod connects straight to the gateway Service, so its physical peer is the
# pod and not the load balancer. A pod's connection is handled by the Envoy on its own node.
# control plain HTTP to an unrouted hostname. It must get the platform's HTTP-to-HTTPS redirect
# (301) before and after the TLS attempts, which proves this pod reaches this Envoy and
# that the plain chain, which carries no fence, still answers.
# In proxy-v1 mode every request sends a valid PROXY preamble before HTTP or TLS;
# otherwise the parser's rejection could be mistaken for the physical-source fence.
# tls HTTPS with the SNI of a public route. Certificate checks are off on purpose: a finished
# handshake is the failure this looks for, whatever certificate came back.
# refused curl 35 or 56 with the TLS handshake never finished (time_appconnect is 0)
Expand Down Expand Up @@ -68,15 +70,16 @@ readonly settle_seconds=20
readonly poll_seconds="${PROBE_POLL_SECONDS:-2}"

usage() {
printf 'Usage: %s --context <kube-context> --run-id <digits> [--attempts N]\n' "$(basename "$0")" >&2
printf 'Usage: %s --context <kube-context> --run-id <digits> --transport <plain|proxy-v1> [--attempts N]\n' "$(basename "$0")" >&2
}

context=''
run_id=''
attempts='10'
transport=''
while [[ "$#" -gt 0 ]]; do
case "$1" in
--context | --run-id | --attempts)
--context | --run-id | --attempts | --transport)
if [[ "$#" -lt 2 || -z "$2" ]]; then
usage
exit 1
Expand All @@ -85,6 +88,7 @@ while [[ "$#" -gt 0 ]]; do
--context) context="$2" ;;
--run-id) run_id="$2" ;;
--attempts) attempts="$2" ;;
--transport) transport="$2" ;;
esac
shift 2
;;
Expand All @@ -94,14 +98,19 @@ while [[ "$#" -gt 0 ]]; do
;;
esac
done
if [[ -z "${context}" ]] || [[ ! "${run_id}" =~ ^[1-9][0-9]{0,19}$ ]]; then
if [[ -z "${context}" ]] || [[ ! "${run_id}" =~ ^[1-9][0-9]{0,19}$ ]] || [[ "${transport}" != plain && "${transport}" != proxy-v1 ]]; then
usage
exit 1
fi
if [[ ! "${attempts}" =~ ^[1-9][0-9]?$ ]] || ((attempts > 30)); then
printf 'Refusing to run: --attempts must be a whole number from 1 to 30.\n' >&2
exit 1
fi
curl_transport_option=''
if [[ "${transport}" == proxy-v1 ]]; then
curl_transport_option='--haproxy-protocol'
fi
readonly curl_transport_option

# Worst case: warm-up, the settle wait, every attempt timing out, the closing control, and pull.
readonly pod_deadline_seconds=$(((warmup_attempts + attempts + 1) * attempt_timeout + warmup_attempts + settle_seconds + 120))
Expand Down Expand Up @@ -147,13 +156,21 @@ fi
# shellcheck disable=SC2317,SC2329
cleanup() {
local rc=$?
local remaining
trap - EXIT INT TERM
if ! kc -n "${probe_namespace}" delete pods,ciliumnetworkpolicies -l "${probe_label}=${run_id}" --ignore-not-found --wait=true --timeout=25s >/dev/null; then
printf 'CLEANUP: FAILED, delete objects labelled %s=%s by hand\n' "${probe_label}" "${run_id}"
summary ""
summary "**Cleanup failed:** delete objects labelled \`${probe_label}=${run_id}\` by hand."
exit 4
fi
remaining="$(kc -n "${probe_namespace}" get pods,ciliumnetworkpolicies -l "${probe_label}=${run_id}" -o name)" || remaining='unknown'
if [[ -n "${remaining}" ]]; then
printf 'CLEANUP: FAILED, absence of run-owned objects was not verified\n'
summary ""
summary "**Cleanup failed:** absence of run-owned objects was not verified."
exit 4
fi
printf 'CLEANUP: done\n'
exit "${rc}"
}
Expand Down Expand Up @@ -239,7 +256,7 @@ spec:
- |
set -u
control() {
curl -s -o /dev/null --max-time "\$TIMEOUT" -H "Host: \$CONTROL_HOST" -w '%{http_code}' "http://\$GATEWAY_IP/" 2>/dev/null || true
curl -s ${curl_transport_option} -o /dev/null --max-time "\$TIMEOUT" -H "Host: \$CONTROL_HOST" -w '%{http_code}' "http://\$GATEWAY_IP/" 2>/dev/null || true
}
i=0
first=''
Expand All @@ -255,7 +272,7 @@ spec:
i=0
while [ "\$i" -lt "\$ATTEMPTS" ]; do
i=\$((i + 1))
code="\$(curl -sk -o /dev/null --max-time "\$TIMEOUT" --resolve "\$TLS_HOST:443:\$GATEWAY_IP" -w '%{http_code} %{time_appconnect}' "https://\$TLS_HOST/" 2>/dev/null)"
code="\$(curl -sk ${curl_transport_option} -o /dev/null --max-time "\$TIMEOUT" --resolve "\$TLS_HOST:443:\$GATEWAY_IP" -w '%{http_code} %{time_appconnect}' "https://\$TLS_HOST/" 2>/dev/null)"
printf 'PROBE tls %s %s\\n' "\$?" "\$code"
done
printf 'PROBE control-after %s\\n' "\$(control)"
Expand Down
133 changes: 133 additions & 0 deletions scripts/tests/gateway-denied-peer-wire/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Exercise the rendered diagnostic with real curl and synthetic loopback peers.
package main

import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)

func serve(listener net.Listener, proxy, secure bool, results chan<- error) {
for {
conn, err := listener.Accept()
if err != nil {
return
}
func() {
defer func() { _ = conn.Close() }()
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
reader := bufio.NewReader(conn)
if proxy {
header, err := reader.ReadString('\n')
if err != nil || !strings.HasPrefix(header, "PROXY TCP4 ") || !strings.HasSuffix(header, "\r\n") || len(strings.Fields(header)) != 6 {
results <- fmt.Errorf("missing valid PROXY preamble")
return
}
}
if secure {
header := make([]byte, 5)
if _, err := io.ReadFull(reader, header); err != nil || header[0] != 22 || header[1] != 3 {
results <- fmt.Errorf("missing TLS ClientHello after transport framing")
return
}
// Deliberately refuse before the handshake, just as a denied physical peer is closed.
results <- nil
return
}
line, err := reader.ReadString('\n')
if err != nil || line != "GET / HTTP/1.1\r\n" {
results <- fmt.Errorf("missing HTTP request after transport framing")
return
}
_, err = io.WriteString(conn, "HTTP/1.1 301 Moved Permanently\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
results <- err
}()
}
}

func run() error {
if len(os.Args) != 3 || os.Args[2] != "plain" && os.Args[2] != "proxy-v1" {
return fmt.Errorf("arguments")
}
httpPeer, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
return err
}
defer func() { _ = httpPeer.Close() }()
tlsPeer, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
return err
}
defer func() { _ = tlsPeer.Close() }()
results := make(chan error, 32)
go serve(httpPeer, os.Args[2] == "proxy-v1", false, results)
go serve(tlsPeer, os.Args[2] == "proxy-v1", true, results)
curl, err := exec.LookPath("curl")
if err != nil {
return err
}
work, err := os.MkdirTemp("", "gateway-denied-wire-")
if err != nil {
return err
}
defer func() { _ = os.RemoveAll(work) }()
wrapper := "#!/bin/sh\nexec \"$WIRE_CURL\" --noproxy '*' --connect-to \"$GATEWAY_IP:80:$WIRE_HTTP\" --connect-to \"$TLS_HOST:443:$WIRE_TLS\" \"$@\"\n"
if err := os.WriteFile(filepath.Join(work, "curl"), []byte(wrapper), 0700); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
command := exec.CommandContext(ctx, "sh", os.Args[1])
command.Env = append(os.Environ(), "PATH="+work+":"+os.Getenv("PATH"), "WIRE_CURL="+curl,
"WIRE_HTTP="+httpPeer.Addr().String(), "WIRE_TLS="+tlsPeer.Addr().String(),
"ATTEMPTS=2", "WARMUP_ATTEMPTS=1", "TIMEOUT=1", "SETTLE_SECONDS=0",
"GATEWAY_IP=192.0.2.1", "TLS_HOST=example.test", "CONTROL_HOST=x.invalid")
output, err := command.CombinedOutput()
if err != nil || !bytes.Contains(output, []byte("PROBE control-before 301\n")) || !bytes.Contains(output, []byte("PROBE control-after 301\n")) {
return fmt.Errorf("native controls failed: %v: %s", err, output)
}
attempts := 0
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
if strings.HasPrefix(line, "PROBE tls ") {
attempts++
if !strings.HasPrefix(line, "PROBE tls 35 000 0.000000") && !strings.HasPrefix(line, "PROBE tls 56 000 0.000000") {
return fmt.Errorf("native TLS refusal: %s", line)
}
}
}
if attempts != 2 {
return fmt.Errorf("native TLS attempt count is %d, want two", attempts)
}
for i := 0; i < 4; i++ {
select {
case err := <-results:
if err != nil {
return err
}
case <-ctx.Done():
return fmt.Errorf("native request count below four")
}
}
select {
case <-results:
return fmt.Errorf("unexpected extra request")
default:
}
fmt.Printf("GATEWAY_DIAGNOSTIC_WIRE=PASS transport=%s requests=4\n", os.Args[2])
return nil
}

func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
Loading
Loading