From ac8bd153986bd33d289c2919ff892a76ffbaf677 Mon Sep 17 00:00:00 2001 From: Emilia Desch Date: Wed, 19 Aug 2026 13:55:53 +0200 Subject: [PATCH 1/3] Add NAT64/DNS64 support for IPv6-only clusters on IPv4-only hosts Allow running an IPv6-only OpenShift cluster (IP_STACK=v6) on an IPv4-only host (HOST_IP_STACK=v4) by adding NAT64 (TAYGA) and DNS64 (unbound) plumbing, enabled with ENABLE_NAT64=true. nat64.sh provides the helpers, sourced by 02_configure_host.sh: bring IPv6 up on the baremetal bridge, configure the TAYGA NAT64 tunnel, run an unbound DNS64 resolver (dns64-synthall, since the host has no native IPv6 egress) and point the host and every cluster-facing libvirt network at it, and rewrite node BMC addresses to the IPv6 baremetal address so the IPv6-only in-cluster Ironic can reach them after pivot. common.sh validates the IP_STACK=v6 / HOST_IP_STACK=v4 requirement, 01_install_requirements.sh installs tayga and unbound, and host_cleanup.sh/ocp_cleanup.sh tear the configuration down. --- 01_install_requirements.sh | 13 ++ 02_configure_host.sh | 20 +++ common.sh | 12 ++ config_example.sh | 31 +++++ host_cleanup.sh | 29 ++-- nat64.sh | 272 +++++++++++++++++++++++++++++++++++++ network.sh | 24 +++- ocp_cleanup.sh | 3 +- 8 files changed, 388 insertions(+), 16 deletions(-) create mode 100755 nat64.sh diff --git a/01_install_requirements.sh b/01_install_requirements.sh index 3a6aa0f97..05e17516b 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -207,6 +207,19 @@ if [[ "${NODES_PLATFORM:-}" == "baremetal" ]] ; then sudo dnf -y install ipmitool fi +# Install NAT64 dependencies if enabled +if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then + echo "Installing NAT64 dependencies (TAYGA, unbound)..." + # TAYGA provides the NAT64 translation; unbound provides DNS64 synthesis. + # unbound is used rather than CoreDNS because its built-in dns64 module + # reliably synthesizes AAAA records (the CoreDNS dns64 plugin build did not). + sudo dnf -y install tayga unbound + # We run our own DNS64 unbound instance on a dedicated port; make sure the + # stock unbound.service does not also grab port 53 and clash with the host + # NetworkManager resolver. + sudo systemctl disable --now unbound.service 2>/dev/null || true +fi + retry_with_timeout 5 60 "curl -L $OPENSHIFT_CLIENT_TOOLS_URL | sudo tar -U -C /usr/local/bin -xzf -" sudo chmod +x /usr/local/bin/oc oc version --client -o json diff --git a/02_configure_host.sh b/02_configure_host.sh index a23dc9997..daa120493 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -4,6 +4,7 @@ set -euxo pipefail source logging.sh source common.sh source network.sh +source nat64.sh source utils.sh source validation.sh source oc_mirror.sh @@ -293,6 +294,13 @@ if [ "${NUM_EXTRA_WORKERS}" -ne 0 ] || [ "${NUM_ARM_WORKERS}" -ne 0 ]; then fi fi +# For NAT64 (IPv6-only cluster on an IPv4-only host) the BMC emulator must be +# reached over IPv6 by the in-cluster Ironic pods; rewrite the node BMC addresses +# to the host's IPv6 baremetal address. +if [[ "${ENABLE_NAT64}" == "true" ]]; then + nat64_fixup_bmc_addresses +fi + # shellcheck disable=SC2034 ZONE="\nZONE=libvirt" @@ -474,6 +482,12 @@ if [ "$EXT_IF" ]; then sudo $IPTABLES -A FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT fi +# When NAT64 is enabled, ensure IPv6 forwarding rules are set for the bridge +if [[ "${ENABLE_NAT64}" == "true" ]]; then + sudo ip6tables -A FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT 2>/dev/null || true + sudo ip6tables -A FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true +fi + # Switch NetworkManager to internal DNS if [ "$MANAGE_BR_BRIDGE" == "y" ]; then switch_to_internal_dns @@ -536,6 +550,12 @@ fi sudo virsh net-list | grep "${PROVISIONING_NETWORK_NAME}" || sudo virsh net-start "${PROVISIONING_NETWORK_NAME}" sudo virsh net-list | grep "${BAREMETAL_NETWORK_NAME}" || sudo virsh net-start "${BAREMETAL_NETWORK_NAME}" +# Configure NAT64/DNS64 if enabled +if [[ "${ENABLE_NAT64}" == "true" ]]; then + configure_nat64_bridge_ipv6 + configure_tayga + configure_dns64 +fi # Setup a single nfs export for image registry if [ "${PERSISTENT_IMAGEREG}" == true ] ; then diff --git a/common.sh b/common.sh index c7049c3d5..627864e50 100644 --- a/common.sh +++ b/common.sh @@ -433,6 +433,18 @@ if [[ "${BOOTSTRAP_IN_PLACE}" == "true" ]]; then export NETWORK_TYPE="OVNKubernetes" fi +# Validate NAT64 configuration +if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then + if [[ "${IP_STACK:-v6}" != "v6" ]]; then + error "ENABLE_NAT64=true requires IP_STACK=v6 (got IP_STACK=${IP_STACK:-v6})" + exit 1 + fi + if [[ "${HOST_IP_STACK:-${IP_STACK:-v6}}" != "v4" ]]; then + error "ENABLE_NAT64=true requires HOST_IP_STACK=v4 (got HOST_IP_STACK=${HOST_IP_STACK:-${IP_STACK:-v6}})" + exit 1 + fi +fi + # Defaults the DISABLE_MULTICAST variable export DISABLE_MULTICAST=${DISABLE_MULTICAST:-false} diff --git a/config_example.sh b/config_example.sh index 16a546f78..16a9fda77 100755 --- a/config_example.sh +++ b/config_example.sh @@ -441,6 +441,37 @@ set -x #export EXTERNAL_SUBNET_V4="192.168.111.0/24" #export EXTERNAL_SUBNET_V6="fd2e:6f44:5dd8:c956::/120" +# ENABLE_NAT64 - +# Enable NAT64/DNS64 to allow IPv6-only clusters (IP_STACK=v6) to run on +# IPv4-only hosts (HOST_IP_STACK=v4). Uses TAYGA for NAT64 translation and +# CoreDNS for DNS64 synthesis, enabling cluster VMs to reach external IPv4 +# resources (e.g. container registries) via synthesized IPv6 addresses. +# Requires: IP_STACK=v6 and HOST_IP_STACK=v4 +# Default: false +# +#export ENABLE_NAT64=true + +# NAT64_PREFIX - +# IPv6 prefix used for NAT64 address translation. Packets sent to addresses +# in this prefix are translated to IPv4 by TAYGA. +# Default: "64:ff9b::/96" +# Note: The well-known prefix 64:ff9b::/96 cannot reach RFC1918 addresses. +# Use a ULA prefix (e.g. "fd00:64::/96") if you need to reach private IPv4. +# +#export NAT64_PREFIX="64:ff9b::/96" + +# NAT64_V4_POOL - +# IPv4 address pool used by TAYGA for dynamic NAT64 mappings. +# Default: "192.168.255.0/24" +# +#export NAT64_V4_POOL="192.168.255.0/24" + +# NAT64_V4_ADDR - +# TAYGA's own IPv4 address on the NAT64 TUN interface. +# Default: "192.168.255.1" +# +#export NAT64_V4_ADDR="192.168.255.1" + # ENABLE_BOOTSTRAP_STATIC_IP - # Configure a static IP for the bootstrap VM external NIC # (Currently this just expects a non-empty value, the IP is fixed to .9) diff --git a/host_cleanup.sh b/host_cleanup.sh index 047379338..3b38e3f32 100755 --- a/host_cleanup.sh +++ b/host_cleanup.sh @@ -3,6 +3,8 @@ set -x source logging.sh source common.sh +source network.sh +source nat64.sh source utils.sh source validation.sh @@ -29,10 +31,15 @@ ansible-playbook \ -e "virthost=$HOSTNAME" \ -e "manage_baremetal=$MANAGE_BR_BRIDGE" \ -e "nodes_file=$NODES_FILE" \ - -i "${VM_SETUP_PATH}/inventory.ini" \ - -b -vvv "${VM_SETUP_PATH}/teardown-playbook.yml" + -i ${VM_SETUP_PATH}/inventory.ini \ + -b -vvv ${VM_SETUP_PATH}/teardown-playbook.yml -sudo rm -rf "/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf" /etc/yum.repos.d/delorean* +# Clean up NAT64/DNS64 if enabled +if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then + cleanup_nat64 +fi + +sudo rm -rf /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf /etc/yum.repos.d/delorean* sudo rm -rf /etc/NetworkManager/conf.d/dnsmasq.conf sudo rm -rf /etc/NetworkManager/dnsmasq.d/upstream.conf if systemctl is-active --quiet NetworkManager; then @@ -44,8 +51,8 @@ fi # handle upgrade from legacy network scripts for interface in ${PROVISIONING_NETWORK_NAME} ${BAREMETAL_NETWORK_NAME} ${PRO_IF} ${INT_IF}; do interface_config=/etc/sysconfig/network-scripts/ifcfg-${interface} - if [ -e "$interface_config" ]; then - sudo rm -f "$interface_config" + if [ -e $interface_config ]; then + sudo rm -f $interface_config IF_FOUND=true fi done @@ -57,22 +64,22 @@ fi # There was a bug in this file, it may need to be recreated. # delete the interface as it can cause issues when not rebooting if [ "$MANAGE_PRO_BRIDGE" == "y" ]; then - sudo nmcli con del "${PROVISIONING_NETWORK_NAME}" || true - sudo ip link delete "${PROVISIONING_NETWORK_NAME}" || true + sudo nmcli con del ${PROVISIONING_NETWORK_NAME} || true + sudo ip link delete ${PROVISIONING_NETWORK_NAME} || true if [[ -d /sys/class/net/pro-ipv6-dummy ]]; then sudo ip link delete pro-ipv6-dummy || true fi - sudo rm -f "/etc/NetworkManager/system-connections/${PROVISIONING_NETWORK_NAME}.nmconnection" + sudo rm -f /etc/NetworkManager/system-connections/${PROVISIONING_NETWORK_NAME}.nmconnection fi # Leaving this around causes issues when the host is rebooted # delete the interface as it can cause issues when not rebooting if [ "$MANAGE_BR_BRIDGE" == "y" ]; then - sudo nmcli con del "${BAREMETAL_NETWORK_NAME}" || true - sudo ip link delete "${BAREMETAL_NETWORK_NAME}" || true + sudo nmcli con del ${BAREMETAL_NETWORK_NAME} || true + sudo ip link delete ${BAREMETAL_NETWORK_NAME} || true if [[ -d /sys/class/net/bm-ipv6-dummy ]]; then sudo ip link delete bm-ipv6-dummy || true fi - sudo rm -f "/etc/NetworkManager/system-connections/${BAREMETAL_NETWORK_NAME}.nmconnection" + sudo rm -f /etc/NetworkManager/system-connections/${BAREMETAL_NETWORK_NAME}.nmconnection fi # Drop all ebtables rules diff --git a/nat64.sh b/nat64.sh new file mode 100755 index 000000000..f2d316793 --- /dev/null +++ b/nat64.sh @@ -0,0 +1,272 @@ +#!/bin/bash +# NAT64/DNS64 helper functions for enabling IPv6-only clusters +# on IPv4-only hosts using TAYGA (NAT64) and unbound (DNS64). + +export NAT64_TAYGA_CONF="/etc/tayga.conf" +export NAT64_TAYGA_DATA_DIR="/var/db/tayga" +export NAT64_TUN_INTERFACE="nat64" +export NAT64_DNS64_PORT="5353" +export NAT64_UNBOUND_CONF="/etc/unbound/unbound-dns64.conf" +export NAT64_UNBOUND_SERVICE="unbound-dns64" +export NAT64_DNSMASQ_CONF="/etc/NetworkManager/dnsmasq.d/nat64-dns64.conf" + +function configure_nat64_bridge_ipv6() { + local bridge_name="${BAREMETAL_NETWORK_NAME}" + local ipv6_addr + ipv6_addr=$(nth_ip "$EXTERNAL_SUBNET_V6" 1) + + echo "Configuring IPv6 on bridge ${bridge_name} for NAT64..." + + # Enable IPv6 forwarding + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + sudo sysctl -w net.ipv6.conf.default.forwarding=1 + + # Add IPv6 address to the baremetal bridge + local prefix_len + prefix_len=$(echo "${EXTERNAL_SUBNET_V6}" | cut -d'/' -f2) + sudo ip -6 addr add "${ipv6_addr}/${prefix_len}" dev "${bridge_name}" 2>/dev/null || true + + echo "IPv6 address ${ipv6_addr}/${prefix_len} configured on ${bridge_name}" +} + +function configure_tayga() { + echo "Configuring TAYGA NAT64 daemon..." + + # Create TAYGA data directory + sudo mkdir -p "${NAT64_TAYGA_DATA_DIR}" + + # Write TAYGA configuration + sudo tee "${NAT64_TAYGA_CONF}" > /dev/null </dev/null || true + + # Add IPv6 route for the NAT64 prefix via the TUN interface + sudo ip -6 route add "${NAT64_PREFIX}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + + # Enable IPv4 forwarding + sudo sysctl -w net.ipv4.ip_forward=1 + + # Set up iptables MASQUERADE for translated traffic + sudo iptables -t nat -C POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE 2>/dev/null || \ + sudo iptables -t nat -A POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE + + # Start TAYGA daemon + sudo tayga + + echo "TAYGA NAT64 daemon started (prefix=${NAT64_PREFIX}, pool=${NAT64_V4_POOL})" +} + +function configure_dns64() { + echo "Configuring unbound for DNS64..." + + # Remove any legacy CoreDNS DNS64 service from before the switch to unbound so + # it does not hold the DNS64 port. + _nat64_remove_legacy_coredns + + # Discover the host's real upstream resolvers. NAT64 hosts frequently cannot + # reach public resolvers (e.g. 8.8.8.8) through their firewall, so forward + # DNS64 queries to whatever resolvers the host itself uses. When NetworkManager + # runs in dnsmasq mode /etc/resolv.conf points at a loopback stub and the real + # upstream servers live in no-stub-resolv.conf. + local upstreams + upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /run/NetworkManager/no-stub-resolv.conf 2>/dev/null) + if [[ -z "${upstreams//[[:space:]]/}" ]]; then + upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /etc/resolv.conf 2>/dev/null) + fi + upstreams=${upstreams:-8.8.8.8} + echo "DNS64 upstream resolvers: ${upstreams}" + + # Write the unbound DNS64 config. dns64-synthall makes unbound synthesize an + # AAAA in ${NAT64_PREFIX} for EVERY name, even ones that already have a native + # AAAA: the host has no native IPv6 egress, so all IPv6 must be routed through + # NAT64/TAYGA. (The CoreDNS dns64 plugin was used previously but its build did + # not synthesize at all, so IPv6-only nodes received unreachable native AAAA.) + sudo mkdir -p "$(dirname "${NAT64_UNBOUND_CONF}")" + sudo tee "${NAT64_UNBOUND_CONF}" > /dev/null < /dev/null < /dev/null </dev/null 2>&1 || return 0 + + echo "Pointing libvirt network ${net} dnsmasq at DNS64 resolver..." + sudo virsh net-dumpxml "${net}" > "${xml}" + + # Idempotent: only inject the forwarding options if not already present. + if ! grep -q "server=127.0.0.1#${NAT64_DNS64_PORT}" "${xml}"; then + sudo sed -i "/<\/dnsmasq:options>/i\\ + \\ + " "${xml}" + fi + + sudo virsh net-destroy "${net}" + sudo virsh net-undefine "${net}" + sudo virsh net-define "${xml}" + sudo virsh net-start "${net}" + + # net-destroy drops the bridge; restore a dummy for carrier and addr_gen_mode=0 + # so the network's IPv6 address comes up before the VMs provide carrier (needed + # for IPv6 on EL9). + sudo ip link add name "${net}-dmy" up master "${net}" type dummy 2>/dev/null || true + echo 0 | sudo dd of="/proc/sys/net/ipv6/conf/${net}/addr_gen_mode" 2>/dev/null || true +} + +# When running an IPv6-only cluster via NAT64, the in-cluster Ironic runs as +# IPv6-only pods (hostNetwork on v6-only nodes) and cannot reach the sushy/redfish +# BMC emulator at its IPv4 baremetal address. Rewrite the generated node BMC +# addresses to the host's IPv6 baremetal address (which sushy also listens on) so +# both the bootstrap Ironic and the pivoted in-cluster Ironic can control the nodes. +function nat64_fixup_bmc_addresses() { + local v4host v6host + v4host=$(nth_ip "${EXTERNAL_SUBNET_V4}" 1) + v6host=$(nth_ip "${EXTERNAL_SUBNET_V6}" 1) + if [[ -z "${v4host}" || -z "${v6host}" ]]; then + echo "nat64_fixup_bmc_addresses: missing v4/v6 baremetal host address, skipping" + return 0 + fi + echo "Rewriting node BMC addresses from ${v4host} to [${v6host}] for NAT64..." + local f tmp + for f in "${NODES_FILE}" "${NODES_FILE}.orig" "${EXTRA_NODES_FILE:-}" "${ARM_NODES_FILE:-}"; do + [[ -n "${f}" && -f "${f}" ]] || continue + tmp="${f}.nat64" + # Literal (non-regex) host replacement via split/join on the "//host:" token. + jq --arg old "//${v4host}:" --arg new "//[${v6host}]:" \ + '(.nodes[]?.driver_info.address) |= (. / $old | join($new))' \ + "${f}" > "${tmp}" && mv "${tmp}" "${f}" + done +} + +# Remove the legacy CoreDNS DNS64 service (superseded by unbound). No-op if absent. +function _nat64_remove_legacy_coredns() { + if sudo systemctl is-active --quiet coredns-nat64.service 2>/dev/null; then + sudo systemctl stop coredns-nat64.service + fi + sudo systemctl disable coredns-nat64.service 2>/dev/null || true + sudo rm -f /etc/systemd/system/coredns-nat64.service + sudo rm -rf /etc/coredns + sudo systemctl daemon-reload +} + +function cleanup_nat64() { + echo "Cleaning up NAT64/DNS64 configuration..." + + # Remove any legacy CoreDNS DNS64 service from before the switch to unbound + _nat64_remove_legacy_coredns + + # Stop and disable the DNS64 unbound instance + if sudo systemctl is-active --quiet "${NAT64_UNBOUND_SERVICE}.service" 2>/dev/null; then + sudo systemctl stop "${NAT64_UNBOUND_SERVICE}.service" + fi + sudo systemctl disable "${NAT64_UNBOUND_SERVICE}.service" 2>/dev/null || true + sudo rm -f "/etc/systemd/system/${NAT64_UNBOUND_SERVICE}.service" + sudo systemctl daemon-reload + + # Remove unbound DNS64 configuration + sudo rm -f "${NAT64_UNBOUND_CONF}" + + # Remove the per-network DNS64 dummy carrier interfaces + local net + for net in "${BAREMETAL_NETWORK_NAME}" ${EXTRA_NETWORK_NAMES:-}; do + sudo ip link del "${net}-dmy" 2>/dev/null || true + done + # Remove the legacy baremetal dummy name used by earlier revisions + sudo ip link del bm-ipv6-dummy 2>/dev/null || true + + # Remove dnsmasq DNS64 forwarding config + sudo rm -f "${NAT64_DNSMASQ_CONF}" + if systemctl is-active --quiet NetworkManager; then + sudo systemctl reload NetworkManager + fi + + # Stop TAYGA + sudo tayga --rmtun 2>/dev/null || true + sudo pkill -f "^tayga" 2>/dev/null || true + + # Remove NAT64 routes + sudo ip route del "${NAT64_V4_POOL}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + sudo ip -6 route del "${NAT64_PREFIX}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + + # Remove iptables masquerade rule + sudo iptables -t nat -D POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE 2>/dev/null || true + + # Clean up TAYGA data and config + sudo rm -rf "${NAT64_TAYGA_DATA_DIR}" + sudo rm -f "${NAT64_TAYGA_CONF}" + + echo "NAT64/DNS64 cleanup complete" +} diff --git a/network.sh b/network.sh index 97204e389..a8c930717 100755 --- a/network.sh +++ b/network.sh @@ -32,6 +32,12 @@ export PATH_CONF_DNSMASQ="/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME export IP_STACK=${IP_STACK:-"v6"} export HOST_IP_STACK=${HOST_IP_STACK:-${IP_STACK}} +# NAT64/DNS64 configuration for IPv6-only clusters on IPv4 hosts +export ENABLE_NAT64=${ENABLE_NAT64:-false} +export NAT64_PREFIX=${NAT64_PREFIX:-"64:ff9b::/96"} +export NAT64_V4_POOL=${NAT64_V4_POOL:-"192.168.255.0/24"} +export NAT64_V4_ADDR=${NAT64_V4_ADDR:-"192.168.255.1"} + # Record the pre-defaulting NETWORK_TYPE export ORIG_NETWORK_TYPE=${NETWORK_TYPE:-""} @@ -96,7 +102,12 @@ if [[ "$HOST_IP_STACK" = "v4" ]] then export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-"172.22.0.0/24"} export EXTERNAL_SUBNET_V4=${EXTERNAL_SUBNET_V4:-"192.168.111.0/24"} - export EXTERNAL_SUBNET_V6="" + if [[ "${ENABLE_NAT64}" == "true" ]]; then + # NAT64: bridge needs IPv6 for cluster VMs even though host is IPv4-only + export EXTERNAL_SUBNET_V6=${EXTERNAL_SUBNET_V6:-"fd2e:6f44:5dd8:c956::/120"} + else + export EXTERNAL_SUBNET_V6="" + fi elif [[ "$HOST_IP_STACK" = "v6" ]]; then export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-"fd00:1101::0/64"} export EXTERNAL_SUBNET_V4="" @@ -114,6 +125,10 @@ else exit 1 fi +if [[ "${ENABLE_NAT64}" == "true" ]]; then + export NAT64_V6_ADDR=${NAT64_V6_ADDR:-$(nth_ip "$EXTERNAL_SUBNET_V6" 3)} +fi + function openshift_sdn_deprecated() { # OpenShiftSDN is deprecated in 4.15 and later printf '4.15\n%s\n' "$(openshift_version)" | sort -V -C @@ -140,7 +155,8 @@ elif [[ "$IP_STACK" = "v6" ]]; then export SERVICE_SUBNET_V4="" export SERVICE_SUBNET_V6=${SERVICE_SUBNET_V6:-"fd02::/112"} export NETWORK_TYPE=${NETWORK_TYPE:-"OVNKubernetes"} - if [[ ${AGENT_E2E_TEST_BOOT_MODE} != "ISO_NO_REGISTRY" ]]; then + if [[ "${ENABLE_NAT64}" != "true" ]] && [[ ${AGENT_E2E_TEST_BOOT_MODE} != "ISO_NO_REGISTRY" ]]; then + # NAT64 provides external registry access, so mirroring is not required export MIRROR_IMAGES=${MIRROR_IMAGES:-true} fi elif [[ "$IP_STACK" = "v4v6" || "$IP_STACK" = "v6v4" ]]; then @@ -256,7 +272,7 @@ function get_vips() { # Returns: # None # - if [[ -n "${EXTERNAL_SUBNET_V4}" ]]; then + if [[ -n "${EXTERNAL_SUBNET_V4}" ]] && [[ "${IP_STACK}" != "v6" ]]; then API_VIPS_V4=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @"$(network_ip "${BAREMETAL_NETWORK_NAME}" v4)" | awk '{print $NF}') if [ -z "$EXTERNAL_LOADBALANCER" ]; then INGRESS_VIPS_V4=$(nth_ip "$EXTERNAL_SUBNET_V4" 4) @@ -265,7 +281,7 @@ function get_vips() { fi fi - if [[ -n "${EXTERNAL_SUBNET_V6}" ]]; then + if [[ -n "${EXTERNAL_SUBNET_V6}" ]] && [[ "${IP_STACK}" != "v4" ]]; then API_VIPS_V6=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @"$(network_ip "${BAREMETAL_NETWORK_NAME}" v6)" | awk '{print $NF}') if [ -z "$EXTERNAL_LOADBALANCER" ]; then INGRESS_VIPS_V6=$(nth_ip "$EXTERNAL_SUBNET_V6" 4) diff --git a/ocp_cleanup.sh b/ocp_cleanup.sh index d6645d8f8..5bdfda7c1 100755 --- a/ocp_cleanup.sh +++ b/ocp_cleanup.sh @@ -18,7 +18,8 @@ if [ -d "${OCP_DIR}" ]; then sudo rm -rf "${OCP_DIR}" fi -sudo rm -rf "/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf" +sudo rm -rf /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf +sudo rm -f /etc/NetworkManager/dnsmasq.d/nat64-dns64.conf # Cleanup ssh keys for baremetal network if [ -f "$HOME/.ssh/known_hosts" ]; then From 28a095eadda3a55d7807c95e2505d2d16c55aba9 Mon Sep 17 00:00:00 2001 From: Emilia Desch Date: Wed, 19 Aug 2026 13:56:07 +0200 Subject: [PATCH 2/3] NAT64: make sushy BMC cert valid for the IPv6 baremetal address metal3-dev-env only puts the IPv4 baremetal address in the sushy-tools BMC emulator certificate SAN. With NAT64 the in-cluster Ironic pods are IPv6-only and reach the BMC over IPv6, and on OCP >= 4.22 dev-scripts no longer emits disableCertificateVerification, so certificate verification is always on and the IPv6 redfish connection is rejected with an IP address mismatch. Add nat64_fixup_sushy_cert (called from 02_configure_host.sh before step 05 embeds the cert into the install-config trust bundle) to regenerate the cert with both the IPv4 and IPv6 SANs, reusing the existing key, and restart sushy-tools. Idempotent and a no-op when sushy or the IPv6 address is absent. --- 02_configure_host.sh | 4 ++++ nat64.sh | 54 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/02_configure_host.sh b/02_configure_host.sh index daa120493..32fa769ff 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -555,6 +555,10 @@ if [[ "${ENABLE_NAT64}" == "true" ]]; then configure_nat64_bridge_ipv6 configure_tayga configure_dns64 + # Make the sushy BMC cert valid for the IPv6 baremetal address so the + # IPv6-only in-cluster Ironic can reach the BMC. Must run before step 05 + # embeds this cert into the install-config trust bundle. + nat64_fixup_sushy_cert fi # Setup a single nfs export for image registry diff --git a/nat64.sh b/nat64.sh index f2d316793..a44ed0736 100755 --- a/nat64.sh +++ b/nat64.sh @@ -211,6 +211,60 @@ function nat64_fixup_bmc_addresses() { done } +# Regenerate the sushy-tools BMC emulator TLS certificate so it is valid for the +# host's IPv6 baremetal address in addition to its IPv4 one. metal3-dev-env only +# puts the IPv4 baremetal address in the cert's SAN, but for NAT64 the in-cluster +# Ironic pods are IPv6-only and must reach the BMC over IPv6; without the IPv6 SAN +# they reject the redfish connection with an "IP address mismatch" TLS error. +# (On OCP >= 4.22 dev-scripts no longer emits disableCertificateVerification, so +# certificate verification is always on and the cert MUST carry the IPv6 SAN.) +# +# This must run before 05_create_install_config.sh, which embeds this cert into the +# install-config trust bundle (see ocp_install_env.sh). The existing key is reused +# and the sushy-tools container is restarted so it serves the new cert. Idempotent +# and a no-op when sushy or the IPv6 address is not present. +function nat64_fixup_sushy_cert() { + local sushy_dir="${WORKING_DIR}/virtualbmc/sushy-tools" + local cert="${sushy_dir}/cert.pem" + local key="${sushy_dir}/key.pem" + local v4host v6host + v4host=$(nth_ip "${EXTERNAL_SUBNET_V4}" 1) + v6host=$(nth_ip "${EXTERNAL_SUBNET_V6}" 1) + if [[ ! -f "${cert}" || ! -f "${key}" || -z "${v6host}" ]]; then + echo "nat64_fixup_sushy_cert: sushy cert/key or IPv6 host address missing, skipping" + return 0 + fi + + # Already valid for the IPv6 address? Nothing to do. + if sudo openssl x509 -in "${cert}" -noout -text 2>/dev/null | grep -qiF "${v6host}"; then + echo "sushy BMC cert already valid for ${v6host}" + return 0 + fi + + echo "Regenerating sushy BMC cert with SANs IP:${v4host}, IP:${v6host} for NAT64..." + local tmp + tmp=$(mktemp -d) + cat > "${tmp}/san.cnf" </dev/null || true + + echo "sushy BMC cert regenerated for ${v4host} and ${v6host}" +} + # Remove the legacy CoreDNS DNS64 service (superseded by unbound). No-op if absent. function _nat64_remove_legacy_coredns() { if sudo systemctl is-active --quiet coredns-nat64.service 2>/dev/null; then From 29cd16527b300d6a6cc2e1f0e57a088b1b0d49d9 Mon Sep 17 00:00:00 2001 From: Emilia Desch Date: Thu, 20 Aug 2026 15:39:50 +0200 Subject: [PATCH 3/3] apply PR review fixes, systemd TAYGA, unbound DNS64, sudo-safe sushy cert + BMC rewrite, idempotent firewall/cleanup, and config validation --- 01_install_requirements.sh | 11 ++- 02_configure_host.sh | 8 +- common.sh | 39 +++++++--- config_example.sh | 24 +++++- host_cleanup.sh | 31 ++++---- nat64.sh | 156 ++++++++++++++++++++++++++++--------- ocp_cleanup.sh | 3 +- 7 files changed, 200 insertions(+), 72 deletions(-) diff --git a/01_install_requirements.sh b/01_install_requirements.sh index 05e17516b..ce99dd6a8 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -110,8 +110,15 @@ case $DISTRO in ;; "rhel10"|"centos10") sudo dnf -y install python3-pip - if sudo subscription-manager identity > /dev/null 2>&1; then - sudo subscription-manager repos --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || true + if [[ $DISTRO == "centos10" ]]; then + sudo dnf config-manager --set-enabled crb + sudo dnf -y install epel-release + elif [[ $DISTRO == "rhel10" ]]; then + if sudo subscription-manager identity > /dev/null 2>&1; then + sudo subscription-manager repos --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || true + fi + # EPEL provides tayga (needed for NAT64); mirror the EL9 rhel handling. + sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm fi sudo ln -s /usr/bin/python3 /usr/bin/python || true PYTHON_DEVEL="python3-devel" diff --git a/02_configure_host.sh b/02_configure_host.sh index 32fa769ff..a93e1afcf 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -484,8 +484,12 @@ fi # When NAT64 is enabled, ensure IPv6 forwarding rules are set for the bridge if [[ "${ENABLE_NAT64}" == "true" ]]; then - sudo ip6tables -A FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT 2>/dev/null || true - sudo ip6tables -A FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true + # Check-then-add so re-running does not accumulate duplicate rules; cleanup_nat64 + # removes these with the matching -D commands. + sudo ip6tables -C FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT 2>/dev/null || \ + sudo ip6tables -A FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT + sudo ip6tables -C FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || \ + sudo ip6tables -A FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT fi # Switch NetworkManager to internal DNS diff --git a/common.sh b/common.sh index 627864e50..275de350d 100644 --- a/common.sh +++ b/common.sh @@ -433,18 +433,6 @@ if [[ "${BOOTSTRAP_IN_PLACE}" == "true" ]]; then export NETWORK_TYPE="OVNKubernetes" fi -# Validate NAT64 configuration -if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then - if [[ "${IP_STACK:-v6}" != "v6" ]]; then - error "ENABLE_NAT64=true requires IP_STACK=v6 (got IP_STACK=${IP_STACK:-v6})" - exit 1 - fi - if [[ "${HOST_IP_STACK:-${IP_STACK:-v6}}" != "v4" ]]; then - error "ENABLE_NAT64=true requires HOST_IP_STACK=v4 (got HOST_IP_STACK=${HOST_IP_STACK:-${IP_STACK:-v6}})" - exit 1 - fi -fi - # Defaults the DISABLE_MULTICAST variable export DISABLE_MULTICAST=${DISABLE_MULTICAST:-false} @@ -644,6 +632,33 @@ if [[ ! -z ${AGENT_E2E_TEST_SCENARIO} ]]; then fi fi +# Validate NAT64 configuration. This must run after the agent scenario parsing +# above, which derives IP_STACK from AGENT_E2E_TEST_SCENARIO; validating earlier +# would only ever see the default IP_STACK and silently pass for v4/dual-stack +# agent scenarios. +if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then + if [[ "${IP_STACK:-v6}" != "v6" ]]; then + error "ENABLE_NAT64=true requires IP_STACK=v6 (got IP_STACK=${IP_STACK:-v6})" + exit 1 + fi + if [[ "${HOST_IP_STACK:-${IP_STACK:-v6}}" != "v4" ]]; then + error "ENABLE_NAT64=true requires HOST_IP_STACK=v4 (got HOST_IP_STACK=${HOST_IP_STACK:-${IP_STACK:-v6}})" + exit 1 + fi + # The IPv6-only in-cluster Ironic can only reach the BMC emulator over IPv6, + # which only sushy/redfish listens on; vbmc (ipmi) binds IPv4 only. Reject the + # ipmi and mixed drivers rather than silently leaving those BMCs unreachable. + if [[ "${BMC_DRIVER}" != "redfish" && "${BMC_DRIVER}" != "redfish-virtualmedia" ]]; then + error "ENABLE_NAT64=true requires a redfish-based BMC_DRIVER (redfish or redfish-virtualmedia), got BMC_DRIVER=${BMC_DRIVER}" + exit 1 + fi + # NAT64 provides external registry access, so image mirroring is skipped and no + # local registry is created. Default MIRROR_IMAGES to false here (before the + # registry-override decision later in this file) so the installer is not pointed + # at a registry that was never stood up. + export MIRROR_IMAGES=${MIRROR_IMAGES:-false} +fi + if [[ ! -z ${AGENT_E2E_TEST_BOOT_MODE} ]]; then case "$AGENT_E2E_TEST_BOOT_MODE" in "ISO" | "PXE" | "DISKIMAGE" | "ISCSI"| "ISO_NO_REGISTRY") diff --git a/config_example.sh b/config_example.sh index 16a9fda77..d8fad1d14 100755 --- a/config_example.sh +++ b/config_example.sh @@ -444,9 +444,11 @@ set -x # ENABLE_NAT64 - # Enable NAT64/DNS64 to allow IPv6-only clusters (IP_STACK=v6) to run on # IPv4-only hosts (HOST_IP_STACK=v4). Uses TAYGA for NAT64 translation and -# CoreDNS for DNS64 synthesis, enabling cluster VMs to reach external IPv4 +# unbound for DNS64 synthesis, enabling cluster VMs to reach external IPv4 # resources (e.g. container registries) via synthesized IPv6 addresses. -# Requires: IP_STACK=v6 and HOST_IP_STACK=v4 +# Requires: IP_STACK=v6, HOST_IP_STACK=v4, and a redfish-based BMC_DRIVER +# (redfish or redfish-virtualmedia) since the IPv6-only in-cluster Ironic can +# only reach the BMC emulator over IPv6 (vbmc/ipmi listens on IPv4 only). # Default: false # #export ENABLE_NAT64=true @@ -472,6 +474,24 @@ set -x # #export NAT64_V4_ADDR="192.168.255.1" +# NAT64_V6_ADDR - +# TAYGA's own IPv6 address (source of the ICMPv6 errors it generates, e.g. +# for PMTUD). Defaults to the 3rd address of EXTERNAL_SUBNET_V6, which is +# on-link on the baremetal bridge. If you carve TAYGA its own subnet, set this +# to an address outside any locally-attached subnet and route it to the NAT64 +# TUN device so error replies are delivered symmetrically. +# Default: nth_ip(EXTERNAL_SUBNET_V6, 3) +# +#export NAT64_V6_ADDR="fd2e:6f44:5dd8:c956::3" + +# NAT64_DNS64_UPSTREAM - +# Space-separated upstream resolver(s) the DNS64 unbound instance forwards to. +# By default these are auto-discovered from the host's resolver config; set this +# explicitly when the host has no usable upstream in resolv.conf (otherwise DNS64 +# falls back to 8.8.8.8, which is usually unreachable on a firewalled NAT64 host). +# +#export NAT64_DNS64_UPSTREAM="10.0.0.53" + # ENABLE_BOOTSTRAP_STATIC_IP - # Configure a static IP for the bootstrap VM external NIC # (Currently this just expects a non-empty value, the IP is fixed to .9) diff --git a/host_cleanup.sh b/host_cleanup.sh index 3b38e3f32..921ff1130 100755 --- a/host_cleanup.sh +++ b/host_cleanup.sh @@ -31,15 +31,16 @@ ansible-playbook \ -e "virthost=$HOSTNAME" \ -e "manage_baremetal=$MANAGE_BR_BRIDGE" \ -e "nodes_file=$NODES_FILE" \ - -i ${VM_SETUP_PATH}/inventory.ini \ - -b -vvv ${VM_SETUP_PATH}/teardown-playbook.yml + -i "${VM_SETUP_PATH}/inventory.ini" \ + -b -vvv "${VM_SETUP_PATH}/teardown-playbook.yml" -# Clean up NAT64/DNS64 if enabled -if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then - cleanup_nat64 -fi +# Always clean up NAT64/DNS64. cleanup_nat64 is fully idempotent (every step is a +# no-op when nothing is present), so run it unconditionally rather than gating on +# the current ENABLE_NAT64 value: otherwise unsetting ENABLE_NAT64 before teardown +# would strand the unbound service, dnsmasq drop-in, TUN device, routes and rules. +cleanup_nat64 -sudo rm -rf /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf /etc/yum.repos.d/delorean* +sudo rm -rf "/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf" /etc/yum.repos.d/delorean* sudo rm -rf /etc/NetworkManager/conf.d/dnsmasq.conf sudo rm -rf /etc/NetworkManager/dnsmasq.d/upstream.conf if systemctl is-active --quiet NetworkManager; then @@ -51,8 +52,8 @@ fi # handle upgrade from legacy network scripts for interface in ${PROVISIONING_NETWORK_NAME} ${BAREMETAL_NETWORK_NAME} ${PRO_IF} ${INT_IF}; do interface_config=/etc/sysconfig/network-scripts/ifcfg-${interface} - if [ -e $interface_config ]; then - sudo rm -f $interface_config + if [ -e "$interface_config" ]; then + sudo rm -f "$interface_config" IF_FOUND=true fi done @@ -64,22 +65,22 @@ fi # There was a bug in this file, it may need to be recreated. # delete the interface as it can cause issues when not rebooting if [ "$MANAGE_PRO_BRIDGE" == "y" ]; then - sudo nmcli con del ${PROVISIONING_NETWORK_NAME} || true - sudo ip link delete ${PROVISIONING_NETWORK_NAME} || true + sudo nmcli con del "${PROVISIONING_NETWORK_NAME}" || true + sudo ip link delete "${PROVISIONING_NETWORK_NAME}" || true if [[ -d /sys/class/net/pro-ipv6-dummy ]]; then sudo ip link delete pro-ipv6-dummy || true fi - sudo rm -f /etc/NetworkManager/system-connections/${PROVISIONING_NETWORK_NAME}.nmconnection + sudo rm -f "/etc/NetworkManager/system-connections/${PROVISIONING_NETWORK_NAME}.nmconnection" fi # Leaving this around causes issues when the host is rebooted # delete the interface as it can cause issues when not rebooting if [ "$MANAGE_BR_BRIDGE" == "y" ]; then - sudo nmcli con del ${BAREMETAL_NETWORK_NAME} || true - sudo ip link delete ${BAREMETAL_NETWORK_NAME} || true + sudo nmcli con del "${BAREMETAL_NETWORK_NAME}" || true + sudo ip link delete "${BAREMETAL_NETWORK_NAME}" || true if [[ -d /sys/class/net/bm-ipv6-dummy ]]; then sudo ip link delete bm-ipv6-dummy || true fi - sudo rm -f /etc/NetworkManager/system-connections/${BAREMETAL_NETWORK_NAME}.nmconnection + sudo rm -f "/etc/NetworkManager/system-connections/${BAREMETAL_NETWORK_NAME}.nmconnection" fi # Drop all ebtables rules diff --git a/nat64.sh b/nat64.sh index a44ed0736..6d2b2f2cc 100755 --- a/nat64.sh +++ b/nat64.sh @@ -4,6 +4,7 @@ export NAT64_TAYGA_CONF="/etc/tayga.conf" export NAT64_TAYGA_DATA_DIR="/var/db/tayga" +export NAT64_TAYGA_SERVICE="nat64-tayga" export NAT64_TUN_INTERFACE="nat64" export NAT64_DNS64_PORT="5353" export NAT64_UNBOUND_CONF="/etc/unbound/unbound-dns64.conf" @@ -45,27 +46,36 @@ dynamic-pool ${NAT64_V4_POOL} data-dir ${NAT64_TAYGA_DATA_DIR} EOF - # Create the TUN device via TAYGA - sudo tayga --mktun - - # Configure the TUN interface - sudo ip link set "${NAT64_TUN_INTERFACE}" up - - # Add IPv4 route for the NAT64 pool via the TUN interface - sudo ip route add "${NAT64_V4_POOL}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true - - # Add IPv6 route for the NAT64 prefix via the TUN interface - sudo ip -6 route add "${NAT64_PREFIX}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + # Run TAYGA as a systemd service so the translator, its TUN device, routes and + # forwarding survive a host reboot (mirroring the unbound DNS64 service). All + # setup lives in ExecStartPre so it is re-established on every (re)start; the + # '-' prefix and the -C||-A guard keep it idempotent (re-running 02_configure_host + # no longer fails on an already-existing TUN device). + sudo tee "/etc/systemd/system/${NAT64_TAYGA_SERVICE}.service" > /dev/null </dev/null || /usr/sbin/iptables -t nat -A POSTROUTING -s ${NAT64_V4_POOL} -j MASQUERADE' +ExecStart=/usr/sbin/tayga --nodetach -c ${NAT64_TAYGA_CONF} +Restart=on-failure +RestartSec=5 - # Set up iptables MASQUERADE for translated traffic - sudo iptables -t nat -C POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE 2>/dev/null || \ - sudo iptables -t nat -A POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE +[Install] +WantedBy=multi-user.target +EOF - # Start TAYGA daemon - sudo tayga + sudo systemctl daemon-reload + sudo systemctl enable "${NAT64_TAYGA_SERVICE}.service" + # restart (not just start) so a re-run picks up any config/unit changes. + sudo systemctl restart "${NAT64_TAYGA_SERVICE}.service" echo "TAYGA NAT64 daemon started (prefix=${NAT64_PREFIX}, pool=${NAT64_V4_POOL})" } @@ -77,19 +87,37 @@ function configure_dns64() { # it does not hold the DNS64 port. _nat64_remove_legacy_coredns - # Discover the host's real upstream resolvers. NAT64 hosts frequently cannot - # reach public resolvers (e.g. 8.8.8.8) through their firewall, so forward - # DNS64 queries to whatever resolvers the host itself uses. When NetworkManager - # runs in dnsmasq mode /etc/resolv.conf points at a loopback stub and the real - # upstream servers live in no-stub-resolv.conf. - local upstreams - upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /run/NetworkManager/no-stub-resolv.conf 2>/dev/null) + # Determine the upstream resolvers DNS64 forwards to. An explicit + # NAT64_DNS64_UPSTREAM (space-separated) always wins. Otherwise discover the + # host's real resolvers: NAT64 hosts frequently cannot reach public resolvers + # (e.g. 8.8.8.8) through their firewall, so we forward to whatever the host + # itself uses. When NetworkManager runs in dnsmasq mode /etc/resolv.conf points + # at a loopback stub and the real upstream servers live in no-stub-resolv.conf. + local upstreams="${NAT64_DNS64_UPSTREAM:-}" + if [[ -z "${upstreams//[[:space:]]/}" ]]; then + upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /run/NetworkManager/no-stub-resolv.conf 2>/dev/null) + fi if [[ -z "${upstreams//[[:space:]]/}" ]]; then upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /etc/resolv.conf 2>/dev/null) fi - upstreams=${upstreams:-8.8.8.8} + if [[ -z "${upstreams//[[:space:]]/}" ]]; then + # Fall back to a public resolver so DNS64 still comes up, but warn loudly: + # on a firewalled NAT64 host 8.8.8.8 is usually unreachable and every DNS64 + # lookup will time out. Set NAT64_DNS64_UPSTREAM to fix this deliberately. + echo "WARNING: could not discover any host upstream resolver for DNS64;" >&2 + echo "WARNING: falling back to 8.8.8.8, which is often unreachable on a" >&2 + echo "WARNING: firewalled NAT64 host. Set NAT64_DNS64_UPSTREAM to override." >&2 + upstreams=8.8.8.8 + fi echo "DNS64 upstream resolvers: ${upstreams}" + # NOTE: this unbound instance is a non-validating DNS64 forwarder. dns64-synthall + # rewrites AAAA answers into ${NAT64_PREFIX}, which is fundamentally incompatible + # with DNSSEC AAAA validation, and it implicitly trusts the host's upstream + # resolvers (discovered above). That trust model matches the rest of dev-scripts + # (a lab/CI tool on a controlled network); do not treat this resolver as a + # security boundary. + # Write the unbound DNS64 config. dns64-synthall makes unbound synthesize an # AAAA in ${NAT64_PREFIX} for EVERY name, even ones that already have a native # AAAA: the host has no native IPv6 egress, so all IPv6 must be routed through @@ -155,6 +183,15 @@ EOF echo "unbound DNS64 configured on port ${NAT64_DNS64_PORT}" } +# Bounded dummy-interface name for a libvirt network. Linux caps interface names +# at 15 chars (IFNAMSIZ-1); a long ${net} would overflow "${net}-dmy", the create +# would fail and the bridge would be left without carrier (so its IPv6 never comes +# up). Truncate ${net} so the "-dmy" suffix always fits. Used by both the create +# and cleanup paths so they always agree on the name. +function _nat64_dummy_ifname() { + echo "${1:0:11}-dmy" +} + # Rewrite a libvirt network's dnsmasq so it forwards exclusively to the DNS64 # resolver, then recreate the bridge carrier/addr_gen so its IPv6 (and thus DNS) # is up before the VMs boot. Safe to call for a network that does not exist. @@ -177,12 +214,17 @@ function _nat64_point_libvirt_dns64() { sudo virsh net-destroy "${net}" sudo virsh net-undefine "${net}" sudo virsh net-define "${xml}" + # net-undefine clears the autostart flag metal3-dev-env set; restore it so the + # network still comes up after a host reboot. + sudo virsh net-autostart "${net}" sudo virsh net-start "${net}" # net-destroy drops the bridge; restore a dummy for carrier and addr_gen_mode=0 # so the network's IPv6 address comes up before the VMs provide carrier (needed # for IPv6 on EL9). - sudo ip link add name "${net}-dmy" up master "${net}" type dummy 2>/dev/null || true + local dmy + dmy=$(_nat64_dummy_ifname "${net}") + sudo ip link add name "${dmy}" up master "${net}" type dummy 2>/dev/null || true echo 0 | sudo dd of="/proc/sys/net/ipv6/conf/${net}/addr_gen_mode" 2>/dev/null || true } @@ -205,9 +247,22 @@ function nat64_fixup_bmc_addresses() { [[ -n "${f}" && -f "${f}" ]] || continue tmp="${f}.nat64" # Literal (non-regex) host replacement via split/join on the "//host:" token. - jq --arg old "//${v4host}:" --arg new "//[${v6host}]:" \ - '(.nodes[]?.driver_info.address) |= (. / $old | join($new))' \ - "${f}" > "${tmp}" && mv "${tmp}" "${f}" + # select(.address? != null) skips nodes without a BMC address (e.g. a + # manually-edited inventory) instead of aborting jq on a null/string divide. + # On success copy back through the original file (preserving its mode/owner + # rather than replacing it with a fresh-umask temp via mv); always remove the + # temp so a jq failure does not strand a partial ".nat64" file. + if jq --arg old "//${v4host}:" --arg new "//[${v6host}]:" \ + '(.nodes[]?.driver_info | select(.address? != null) | .address) |= (. / $old | join($new))' \ + "${f}" > "${tmp}"; then + # Overwrite in place with sudo: ${f} may be root-owned (written by an + # earlier privileged step) while its directory is user-writable, so a + # plain ">" redirect fails with EACCES. cp onto the existing file also + # keeps its original mode/owner rather than replacing it with a + # fresh-umask temp (which a plain "mv" would have done). + sudo cp "${tmp}" "${f}" + fi + rm -f "${tmp}" done } @@ -230,13 +285,21 @@ function nat64_fixup_sushy_cert() { local v4host v6host v4host=$(nth_ip "${EXTERNAL_SUBNET_V4}" 1) v6host=$(nth_ip "${EXTERNAL_SUBNET_V6}" 1) - if [[ ! -f "${cert}" || ! -f "${key}" || -z "${v6host}" ]]; then + # Probe for the cert/key under sudo: ${WORKING_DIR}/virtualbmc is root-owned and + # root-only (drwxr-x---), so the invoking user cannot traverse it and a plain + # "[[ -f ]]" would report the files missing even when they exist, silently + # skipping the fixup (leaving sushy serving an IPv4-only cert that the IPv6-only + # in-cluster Ironic rejects). The rest of this function already uses sudo. + if ! sudo test -f "${cert}" || ! sudo test -f "${key}" || [[ -z "${v6host}" ]]; then echo "nat64_fixup_sushy_cert: sushy cert/key or IPv6 host address missing, skipping" return 0 fi - # Already valid for the IPv6 address? Nothing to do. - if sudo openssl x509 -in "${cert}" -noout -text 2>/dev/null | grep -qiF "${v6host}"; then + # Already valid for the IPv6 address? Nothing to do. Use openssl's own SAN + # matching (-checkip) rather than grepping the text dump: openssl renders IPv6 + # SANs in uncompressed form, so a substring match for the compressed address + # never matched and the cert was needlessly regenerated on every run. + if sudo openssl x509 -in "${cert}" -noout -checkip "${v6host}" >/dev/null 2>&1; then echo "sushy BMC cert already valid for ${v6host}" return 0 fi @@ -272,7 +335,10 @@ function _nat64_remove_legacy_coredns() { fi sudo systemctl disable coredns-nat64.service 2>/dev/null || true sudo rm -f /etc/systemd/system/coredns-nat64.service - sudo rm -rf /etc/coredns + # NOTE: deliberately do NOT "rm -rf /etc/coredns" here. This helper runs on + # every configure_dns64 call, and /etc/coredns may hold an unrelated host-owned + # CoreDNS configuration. Stopping/removing our own service is enough to free the + # DNS64 port. sudo systemctl daemon-reload } @@ -296,7 +362,7 @@ function cleanup_nat64() { # Remove the per-network DNS64 dummy carrier interfaces local net for net in "${BAREMETAL_NETWORK_NAME}" ${EXTRA_NETWORK_NAMES:-}; do - sudo ip link del "${net}-dmy" 2>/dev/null || true + sudo ip link del "$(_nat64_dummy_ifname "${net}")" 2>/dev/null || true done # Remove the legacy baremetal dummy name used by earlier revisions sudo ip link del bm-ipv6-dummy 2>/dev/null || true @@ -307,9 +373,21 @@ function cleanup_nat64() { sudo systemctl reload NetworkManager fi - # Stop TAYGA + # Stop and disable the TAYGA systemd service + if sudo systemctl is-active --quiet "${NAT64_TAYGA_SERVICE}.service" 2>/dev/null; then + sudo systemctl stop "${NAT64_TAYGA_SERVICE}.service" + fi + sudo systemctl disable "${NAT64_TAYGA_SERVICE}.service" 2>/dev/null || true + sudo rm -f "/etc/systemd/system/${NAT64_TAYGA_SERVICE}.service" + sudo systemctl daemon-reload + # Kill any stray bare tayga daemon FIRST, then tear down the TUN device: tayga + # holds the TUN open, so --rmtun before the process is gone leaves it orphaned. + # Match by exact process name (the cmdline is "/usr/sbin/tayga ...", so an + # "^tayga" -f pattern never matched). Delete the device explicitly as a backstop + # in case --rmtun fails (e.g. the tayga binary is already gone). + sudo pkill -x tayga 2>/dev/null || true sudo tayga --rmtun 2>/dev/null || true - sudo pkill -f "^tayga" 2>/dev/null || true + sudo ip link del "${NAT64_TUN_INTERFACE}" 2>/dev/null || true # Remove NAT64 routes sudo ip route del "${NAT64_V4_POOL}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true @@ -318,6 +396,10 @@ function cleanup_nat64() { # Remove iptables masquerade rule sudo iptables -t nat -D POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE 2>/dev/null || true + # Remove the IPv6 FORWARD rules added by 02_configure_host.sh for NAT64 + sudo ip6tables -D FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT 2>/dev/null || true + sudo ip6tables -D FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true + # Clean up TAYGA data and config sudo rm -rf "${NAT64_TAYGA_DATA_DIR}" sudo rm -f "${NAT64_TAYGA_CONF}" diff --git a/ocp_cleanup.sh b/ocp_cleanup.sh index 5bdfda7c1..d6645d8f8 100755 --- a/ocp_cleanup.sh +++ b/ocp_cleanup.sh @@ -18,8 +18,7 @@ if [ -d "${OCP_DIR}" ]; then sudo rm -rf "${OCP_DIR}" fi -sudo rm -rf /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf -sudo rm -f /etc/NetworkManager/dnsmasq.d/nat64-dns64.conf +sudo rm -rf "/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf" # Cleanup ssh keys for baremetal network if [ -f "$HOME/.ssh/known_hosts" ]; then