feat(attest): accept several collateral endpoints instead of one

dstack/dstack-attest/src/attestation.rs

@@ -11,10 +11,10 @@ pub const TDX_QUOTE_REPORT_DATA_RANGE: std::ops::Range<usize> = 568..632;
 
 use std::{borrow::Cow, time::SystemTime};
 
+use crate::collateral::{amd_kds_client, PccsClient};
 use anyhow::{anyhow, bail, Context, Result};
 use cc_eventlog::{EventLogVersion, RuntimeEvent, TdxEvent};
 use dcap_qvl::{
-    collateral::CollateralClient,
     quote::{EnclaveReport, Quote, Report, TDReport10, TDReport15},
     verify::VerifiedReport as TdxVerifiedReport,
 };
@@ -57,7 +57,7 @@ pub struct AttestationVerifierConfig {
 
 pub struct AttestationVerifier {
     tdx: dcap_qvl::verify::QuoteVerifier,
-    tdx_collateral: CollateralClient,
+    tdx_collateral: PccsClient,
     gcp_tpm: tpm_qvl::QuoteVerifier,
     aws_nitro_enclave: nsm_qvl::QuoteVerifier,
     aws_nitro_tpm: nsm_qvl::QuoteVerifier,
@@ -130,51 +130,27 @@ impl AttestationVerifier {
                 sev_snp = sev_snp.with_root(product, root);
             }
         }
-        let pccs = config
-            .urls
-            .pccs
-            .as_deref()
-            .filter(|v| !v.trim().is_empty())
-            .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL);
-        let amd_kds = config
-            .urls
-            .amd_kds
-            .as_deref()
-            .filter(|v| !v.trim().is_empty())
-            .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL);
         Ok(Self {
             tdx,
-            tdx_collateral: CollateralClient::with_default_http(pccs)?,
+            tdx_collateral: PccsClient::new(&config.urls.pccs)?,
             gcp_tpm,
             aws_nitro_enclave: nsm(aws_nitro_enclave.as_deref(), "AWS Nitro Enclave")?,
             aws_nitro_tpm: nsm(aws_nitro_tpm.as_deref(), "AWS NitroTPM")?,
             sev_snp,
-            amd_kds: AmdKdsClient::with_base_url(amd_kds)?,
+            amd_kds: amd_kds_client(&config.urls.amd_kds)?,
         })
     }
 
     pub fn new_prod(collateral_urls: Option<&CollateralUrls>) -> Result<Self> {
         let collateral_urls = collateral_urls.cloned().unwrap_or_default();
         Ok(Self {
             tdx: dcap_qvl::verify::QuoteVerifier::new_prod(),
-            tdx_collateral: CollateralClient::with_default_http(
-                collateral_urls
-                    .pccs
-                    .as_deref()
-                    .filter(|url| !url.trim().is_empty())
-                    .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL),
-            )?,
+            tdx_collateral: PccsClient::new(&collateral_urls.pccs)?,
             gcp_tpm: tpm_qvl::QuoteVerifier::new_prod(Platform::Gcp)?,
             aws_nitro_enclave: nsm_qvl::QuoteVerifier::new_prod(),
             aws_nitro_tpm: nsm_qvl::QuoteVerifier::new_prod(),
             sev_snp: sev_snp_qvl::QuoteVerifier::new_prod(),
-            amd_kds: AmdKdsClient::with_base_url(
-                collateral_urls
-                    .amd_kds
-                    .as_deref()
-                    .filter(|url| !url.trim().is_empty())
-                    .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL),
-            )?,
+            amd_kds: amd_kds_client(&collateral_urls.amd_kds)?,
         })
     }
 

dstack/dstack-attest/src/collateral.rs

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: © 2024-2025 Phala Network <dstack@phala.network>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Failover across interchangeable collateral endpoints.
+//!
+//! Attestation verification depends on collateral that only the vendor
+//! publishes: Intel's PCCS and AMD's KDS. Both were single points of failure
+//! for verification until now -- one URL each, and no answer meant no
+//! verification. AMD KDS in particular is a single global endpoint with no
+//! official mirror, and it has been unreachable for hours at a time.
+//!
+//! Nothing here relaxes verification. Collateral is signed by the vendor and
+//! checked against roots compiled into the binary, so an endpoint can serve a
+//! stale or absent answer but cannot serve a forged one; which endpoint
+//! answered has no bearing on whether the signature checks pass.
+
+use anyhow::{bail, Context, Result};
+use dcap_qvl::{
+    collateral::{CollateralClient, PHALA_PCCS_URL},
+    verify::VerifiedReport,
+    QuoteCollateralV3,
+};
+use dstack_types::UrlList;
+use sev_snp_qvl::{AmdKdsClient, AMD_KDS_DEFAULT_BASE_URL};
+
+/// Build an AMD KDS client over the configured endpoints, or the vendor
+/// default when none are configured.
+pub(crate) fn amd_kds_client(urls: &UrlList) -> Result<AmdKdsClient> {
+    if urls.is_empty() {
+        return AmdKdsClient::with_base_url(AMD_KDS_DEFAULT_BASE_URL);
+    }
+    AmdKdsClient::with_base_urls(urls.as_slice())
+}
+
+/// Intel PCCS access over one or more interchangeable endpoints.
+///
+/// Endpoints are tried in order and the first success wins.
+///
+/// Unlike the AMD KDS client, this fails over on *any* error rather than only
+/// on ones that look transient. `dcap-qvl` returns `anyhow::Error`, so there is
+/// no status code to classify on, and inventing a classifier by matching error
+/// strings would be worse than the thing it replaces. The cost of the coarser
+/// rule is bounded: a genuinely absent FMSPC costs one extra request per
+/// configured endpoint, and configuring an endpoint list is opt-in.
+#[derive(Clone)]
+pub struct PccsClient {
+    clients: Vec<(String, CollateralClient)>,
+}
+
+impl PccsClient {
+    pub fn new(urls: &UrlList) -> Result<Self> {
+        let urls: Vec<&str> = if urls.is_empty() {
+            vec![PHALA_PCCS_URL]
+        } else {
+            urls.as_slice().iter().map(String::as_str).collect()
+        };
+        let clients = urls
+            .into_iter()
+            .map(|url| {
+                CollateralClient::with_default_http(url)
+                    .with_context(|| format!("failed to create PCCS client for {url}"))
+                    .map(|client| (url.to_string(), client))
+            })
+            .collect::<Result<Vec<_>>>()?;
+        if clients.is_empty() {
+            bail!("PCCS endpoint list is empty");
+        }
+        Ok(Self { clients })
+    }
+
+    pub async fn fetch(&self, quote: &[u8]) -> Result<QuoteCollateralV3> {
+        self.failover("fetch", |client| client.fetch(quote)).await
+    }
+
+    pub async fn fetch_and_verify(&self, quote: &[u8]) -> Result<VerifiedReport> {
+        self.failover("fetch_and_verify", |client| client.fetch_and_verify(quote))
+            .await
+    }
+
+    async fn failover<'a, T, F, Fut>(&'a self, label: &str, mut call: F) -> Result<T>
+    where
+        F: FnMut(&'a CollateralClient) -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        let mut errors: Vec<String> = Vec::new();
+        for (url, client) in &self.clients {
+            match call(client).await {
+                Ok(value) => return Ok(value),
+                Err(err) => errors.push(format!("{url}: {err:#}")),
+            }
+        }
+        match errors.len() {
+            0 => bail!("PCCS {label} had no endpoint to try"),
+            1 => bail!("PCCS {label} failed: {}", errors[0]),
+            n => bail!(
+                "PCCS {label} failed on all {n} endpoints: {}",
+                errors.join("; ")
+            ),
+        }
+    }
+}

dstack/dstack-attest/src/lib.rs

@@ -15,6 +15,7 @@ pub mod amd_sev_snp;
 pub mod attestation;
 #[cfg(feature = "quote")]
 mod aws_nitro_tpm;
+pub mod collateral;
 #[cfg(feature = "quote")]
 mod sev_snp;
 pub mod trust_anchors;

dstack/dstack-types/src/lib.rs

@@ -1238,20 +1238,114 @@ pub struct GatewayClusterConfig {
     pub urls: Vec<String>,
 }
 
+/// One or more interchangeable endpoints for the same collateral service.
+///
+/// Deserializes from either a bare string or a list, so every config file and
+/// every already-serialized `SysConfig` written by an older host keeps parsing
+/// unchanged. Serializes back as a bare string when there is exactly one entry,
+/// for the same reason.
+///
+/// Order is meaningful: callers try entries front to back and keep the first
+/// answer. Blank entries are dropped on the way in, because a config that says
+/// `pccs = ["", "https://..."]` means the second one.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct UrlList(Vec<String>);
+
+impl UrlList {
+    pub fn new(urls: impl IntoIterator<Item = impl Into<String>>) -> Self {
+        Self(
+            urls.into_iter()
+                .map(Into::into)
+                .map(|url| url.trim().to_string())
+                .filter(|url| !url.is_empty())
+                .collect(),
+        )
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    /// The endpoints in the order they should be tried.
+    pub fn as_slice(&self) -> &[String] {
+        &self.0
+    }
+
+    pub fn iter(&self) -> std::slice::Iter<'_, String> {
+        self.0.iter()
+    }
+
+    /// The endpoint a single-URL caller should use. `None` when the list is
+    /// empty, which callers read as "use the platform default".
+    pub fn first(&self) -> Option<&str> {
+        self.0.first().map(String::as_str)
+    }
+}
+
+impl From<String> for UrlList {
+    fn from(url: String) -> Self {
+        Self::new([url])
+    }
+}
+
+impl From<&str> for UrlList {
+    fn from(url: &str) -> Self {
+        Self::new([url])
+    }
+}
+
+impl From<Vec<String>> for UrlList {
+    fn from(urls: Vec<String>) -> Self {
+        Self::new(urls)
+    }
+}
+
+impl Serialize for UrlList {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        match self.0.as_slice() {
+            [single] => serializer.serialize_str(single),
+            many => many.serialize(serializer),
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for UrlList {
+    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum OneOrMany {
+            One(String),
+            Many(Vec<String>),
+        }
+        Ok(match OneOrMany::deserialize(deserializer)? {
+            OneOrMany::One(url) => Self::new([url]),
+            OneOrMany::Many(urls) => Self::new(urls),
+        })
+    }
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub struct CollateralUrls {
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub pccs: Option<String>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub amd_kds: Option<String>,
+    /// Intel PCCS endpoints. Empty means the built-in default.
+    #[serde(default, skip_serializing_if = "UrlList::is_empty")]
+    pub pccs: UrlList,
+    /// AMD KDS endpoints, or mirrors of it. Empty means the built-in default.
+    #[serde(default, skip_serializing_if = "UrlList::is_empty")]
+    pub amd_kds: UrlList,
 }
 
 impl SysConfig {
     pub fn collateral_urls(&self) -> CollateralUrls {
         let mut urls = self.collateral_urls.clone().unwrap_or_default();
-        if urls.pccs.is_none() {
-            urls.pccs.clone_from(&self.legacy_pccs_url);
+        if urls.pccs.is_empty() {
+            if let Some(legacy) = &self.legacy_pccs_url {
+                urls.pccs = UrlList::from(legacy.clone());
+            }
         }
         urls
     }
@@ -2682,3 +2776,88 @@ mod appcompose_sdk_parity {
         );
     }
 }
+
+#[cfg(test)]
+mod collateral_url_tests {
+    use super::*;
+
+    #[test]
+    fn a_bare_string_and_a_list_both_parse() {
+        let single: CollateralUrls =
+            serde_json::from_str(r#"{"pccs":"https://pccs.example"}"#).unwrap();
+        assert_eq!(single.pccs.as_slice(), ["https://pccs.example"]);
+
+        let many: CollateralUrls =
+            serde_json::from_str(r#"{"amd_kds":["https://a/vcek/v1","https://b/vcek/v1"]}"#)
+                .unwrap();
+        assert_eq!(
+            many.amd_kds.as_slice(),
+            ["https://a/vcek/v1", "https://b/vcek/v1"]
+        );
+    }
+
+    /// A `SysConfig` written by a host that predates lists must still round
+    /// trip through a new binary unchanged, because the guest hashes what it
+    /// is given rather than what it would have written itself.
+    #[test]
+    fn a_single_url_serializes_back_as_a_bare_string() {
+        let urls = CollateralUrls {
+            pccs: UrlList::from("https://pccs.example"),
+            amd_kds: UrlList::default(),
+        };
+        assert_eq!(
+            serde_json::to_string(&urls).unwrap(),
+            r#"{"pccs":"https://pccs.example"}"#
+        );
+    }
+
+    #[test]
+    fn several_urls_serialize_as_a_list() {
+        let urls = CollateralUrls {
+            pccs: UrlList::default(),
+            amd_kds: UrlList::new(["https://a", "https://b"]),
+        };
+        assert_eq!(
+            serde_json::to_string(&urls).unwrap(),
+            r#"{"amd_kds":["https://a","https://b"]}"#
+        );
+    }
+
+    #[test]
+    fn blank_entries_are_dropped_on_the_way_in() {
+        let urls: CollateralUrls =
+            serde_json::from_str(r#"{"pccs":["", "  ", "https://real"]}"#).unwrap();
+        assert_eq!(urls.pccs.as_slice(), ["https://real"]);
+
+        let blank: CollateralUrls = serde_json::from_str(r#"{"pccs":"   "}"#).unwrap();
+        assert!(
+            blank.pccs.is_empty(),
+            "a blank string means 'unset', not 'an endpoint named blank'"
+        );
+    }
+
+    #[test]
+    fn the_legacy_pccs_url_field_still_wins_when_no_list_is_configured() {
+        let sys_config: SysConfig = serde_json::from_str(
+            r#"{"kms_urls":[],"gateway_urls":[],"pccs_url":"https://legacy.example","vm_config":"{}"}"#,
+        )
+        .unwrap();
+        assert_eq!(
+            sys_config.collateral_urls().pccs.as_slice(),
+            ["https://legacy.example"]
+        );
+    }
+
+    #[test]
+    fn an_explicit_list_takes_precedence_over_the_legacy_field() {
+        let sys_config: SysConfig = serde_json::from_str(
+            r#"{"kms_urls":[],"gateway_urls":[],"pccs_url":"https://legacy.example",
+                "collateral_urls":{"pccs":["https://new.example"]},"vm_config":"{}"}"#,
+        )
+        .unwrap();
+        assert_eq!(
+            sys_config.collateral_urls().pccs.as_slice(),
+            ["https://new.example"]
+        );
+    }
+}