From e66f4e3ef69851e0ab0ae4e6860632574e020555 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 17 Nov 2026 08:47:04 -0800 Subject: [PATCH 1/4] fix(network): refresh expired policy DNS mappings on connect Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/README.md | 10 + .../src/policy_dns/mod.rs | 307 ++++++++++++++++++ .../src/policy_dns/resolver.rs | 5 +- .../src/policy_dns/runtime.rs | 2 + .../src/policy_dns/store.rs | 91 +++++- .../openshell-supervisor-network/src/proxy.rs | 30 +- docs/how-it-works/policies/network-rules.mdx | 12 +- docs/observability/logging.mdx | 1 + e2e/rust/tests/transparent_tcp.rs | 11 +- 9 files changed, 447 insertions(+), 22 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 92257fb24a..9088bd16a6 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -31,6 +31,16 @@ Without a Workload API socket, token grants fail closed. A process hosting several instances serves `/v1/denials` from a per-instance `SandboxInstanceDenialLog` instead of the shared shorthand log files. +## Policy DNS expiry + +Synthetic identities remain allocated after their DNS mappings expire. New +transparent connections lazily refresh expired, policy-backed mappings through +the same trusted resolver and destination-validation path as sandbox DNS +queries. Refreshes for one address share a lock; failed refreshes back off for +one second and never authorize expired upstream addresses. The service is held +weakly by the store to avoid an ownership cycle. Observation-only mappings, +wrong ports, and mappings from a previous policy generation cannot refresh. + ## Reserved destinations `preauthorize_transparent_open` matches a staged TCP open against the reserved diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index 903f29db2b..27e5ae6163 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -93,6 +93,42 @@ pub(crate) struct PolicyDnsService { trusted_host_gateway: Option, } +#[async_trait::async_trait] +pub(super) trait MappingRefresher: Send + Sync { + async fn refresh(&self, record: &ResolvedEndpointRecord) -> miette::Result<()>; +} + +#[async_trait::async_trait] +impl MappingRefresher for PolicyDnsService { + async fn refresh(&self, record: &ResolvedEndpointRecord) -> miette::Result<()> { + if self.policy.current_generation() != record.policy_generation { + return Err(miette::miette!("policy changed before mapping refresh")); + } + let answer = self + .answer_query( + record.normalized_name.as_str(), + record.family, + Instant::now(), + ) + .await + .map_err(|error| miette::miette!("transparent TCP mapping refresh failed: {error}"))?; + if (answer.address, answer.policy_generation) + != (record.synthetic_address, record.policy_generation) + { + return Err(miette::miette!("policy changed during mapping refresh")); + } + Ok(()) + } +} + +impl PolicyDnsService { + /// Use a weak reference because the service itself owns the mapping store. + fn enable_refresh(self: &Arc) { + let refresher: Arc = self.clone(); + self.store.set_refresher(Arc::downgrade(&refresher)); + } +} + impl PolicyDnsService { pub(crate) fn new( policy: Arc, @@ -114,6 +150,7 @@ impl PolicyDnsService { family: AddressFamily, now: Instant, ) -> Result { + let started = Instant::now(); let normalized_name = NormalizedName::parse(raw_name).map_err(|_| PolicyDnsError::InvalidName)?; if normalized_name.as_str() == crate::policy_local::POLICY_LOCAL_HOST { @@ -190,6 +227,9 @@ impl PolicyDnsService { } }; let ttl = clamp_mapping_ttl(trusted_answer.ttl); + // Start the lease after resolution: a TTL-zero answer must still be + // usable when the trusted resolver takes longer than the TTL floor. + let now = now + started.elapsed(); let allocation_identity = allocation_identity(&eligible); let mut contracts = Vec::new(); for endpoint in eligible { @@ -764,6 +804,273 @@ landlock: { compatibility: best_effort } process: { run_as_user: sandbox, run_as_group: sandbox } "; + struct RefreshResolver { + calls: AtomicUsize, + answer: std::sync::Mutex, + fail: std::sync::atomic::AtomicBool, + pause: std::sync::atomic::AtomicBool, + started: Notify, + release: Notify, + } + + impl TrustedResolver for RefreshResolver { + async fn resolve( + &self, + _name: &NormalizedName, + _family: AddressFamily, + ) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + if self.pause.load(Ordering::SeqCst) { + self.started.notify_one(); + self.release.notified().await; + } + if self.fail.load(Ordering::SeqCst) { + return Err(resolver::ResolveError::NxDomain); + } + Ok(self.answer.lock().unwrap().clone()) + } + } + + fn refresh_service(ttl: Duration) -> Arc> { + let base = service(BASE_POLICY, vec![]); + let service = Arc::new(PolicyDnsService::new( + base.policy, + RefreshResolver { + calls: AtomicUsize::new(0), + answer: std::sync::Mutex::new(TrustedAnswer { + addresses: vec!["8.8.8.8".parse().unwrap()], + ttl, + }), + fail: false.into(), + pause: false.into(), + started: Notify::new(), + release: Notify::new(), + }, + base.store, + None, + )); + service.enable_refresh(); + service + } + + async fn expired_answer(service: &PolicyDnsService) -> SyntheticAnswer { + service + .answer_query( + "db.example", + AddressFamily::Ipv4, + Instant::now().checked_sub(Duration::from_mins(1)).unwrap(), + ) + .await + .unwrap() + } + + #[tokio::test] + async fn expired_mapping_refreshes_same_address_with_new_validated_dns_answers() { + for ttl in [ + Duration::ZERO, + Duration::from_secs(5), + Duration::from_mins(5), + ] { + for sweep in [false, true] { + let service = refresh_service(ttl); + let answer = expired_answer(&service).await; + if sweep { + assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); + } + service.resolver.answer.lock().unwrap().addresses = + vec!["8.8.4.4".parse().unwrap()]; + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + .unwrap(); + let refreshed = service + .store + .lookup( + answer.address, + 5432, + answer.policy_generation, + Instant::now(), + ) + .unwrap(); + assert_eq!( + refreshed.record.contracts[0].pinned_addresses, + ["8.8.4.4".parse::().unwrap()] + ); + assert_ne!(refreshed.record.mapping_id, answer.mapping_id); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + .unwrap(); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); + } + } + } + + #[tokio::test] + async fn expired_mapping_refresh_failures_deny_and_back_off_then_recover() { + let service = refresh_service(Duration::ZERO); + let answer = expired_answer(&service).await; + service.resolver.fail.store(true, Ordering::SeqCst); + for _ in 0..2 { + assert!( + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + .is_err() + ); + } + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); + assert!(matches!( + service.store.lookup( + answer.address, + 5432, + answer.policy_generation, + Instant::now() + ), + Err(MappingLookupError::Expired) + )); + service.resolver.fail.store(false, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(1100)).await; + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + .unwrap(); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn expired_mapping_refresh_rejects_new_forbidden_addresses() { + let service = refresh_service(Duration::ZERO); + let answer = expired_answer(&service).await; + service.resolver.answer.lock().unwrap().addresses = vec!["127.0.0.1".parse().unwrap()]; + assert!( + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + .is_err() + ); + assert!(matches!( + service.store.lookup( + answer.address, + 5432, + answer.policy_generation, + Instant::now() + ), + Err(MappingLookupError::Expired) + )); + } + + #[tokio::test] + async fn expired_mapping_refresh_does_not_upgrade_ports_observations_or_policy_generations() { + let service = refresh_service(Duration::ZERO); + let answer = expired_answer(&service).await; + assert!( + service + .store + .refresh_if_expired(answer.address, 3306, answer.policy_generation) + .await + .is_err() + ); + let observed = service + .answer_query( + "unknown.example", + AddressFamily::Ipv4, + Instant::now().checked_sub(Duration::from_mins(1)).unwrap(), + ) + .await + .unwrap(); + assert!( + service + .store + .refresh_if_expired(observed.address, 5432, observed.policy_generation) + .await + .is_err() + ); + service + .policy + .reload( + include_str!("../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + assert!( + service + .store + .refresh_if_expired(answer.address, 5432, service.policy.current_generation()) + .await + .is_err() + ); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn concurrent_expired_connections_share_one_refresh() { + let service = refresh_service(Duration::ZERO); + let answer = expired_answer(&service).await; + service.resolver.pause.store(true, Ordering::SeqCst); + let mut connects = Vec::new(); + for _ in 0..16 { + let service = service.clone(); + connects.push(tokio::spawn(async move { + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + })); + } + service.resolver.started.notified().await; + // Even a resolver slower than the one-second TTL floor must publish + // a usable lease and satisfy all the waiting connections. + tokio::time::sleep(Duration::from_millis(1100)).await; + service.resolver.pause.store(false, Ordering::SeqCst); + service.resolver.release.notify_one(); + for connect in connects { + connect.await.unwrap().unwrap(); + } + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn policy_revocation_during_expired_mapping_refresh_fails_closed() { + let service = refresh_service(Duration::ZERO); + let answer = expired_answer(&service).await; + service.resolver.pause.store(true, Ordering::SeqCst); + let refreshing = service.clone(); + let connect = tokio::spawn(async move { + refreshing + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + }); + service.resolver.started.notified().await; + service + .policy + .reload( + include_str!("../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + service.resolver.release.notify_one(); + assert!(connect.await.unwrap().is_err()); + assert!( + service + .store + .lookup( + answer.address, + 5432, + service.policy.current_generation(), + Instant::now() + ) + .is_err() + ); + } + #[tokio::test] async fn stages_unknown_name_without_upstream_resolution() { let service = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); diff --git a/crates/openshell-supervisor-network/src/policy_dns/resolver.rs b/crates/openshell-supervisor-network/src/policy_dns/resolver.rs index 34f784ade5..789f720235 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/resolver.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/resolver.rs @@ -78,13 +78,12 @@ pub(crate) enum ResolveError { CnameLimit, } -#[allow(async_fn_in_trait)] pub(crate) trait TrustedResolver: Send + Sync { - async fn resolve( + fn resolve( &self, name: &NormalizedName, family: AddressFamily, - ) -> Result; + ) -> impl Future> + Send; } /// A DNS client pinned to one operator-supplied upstream socket address. diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index d7888f991e..9c1ce7b781 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -112,6 +112,7 @@ impl PolicyDnsRuntime { store.clone(), trusted_host_gateway, )); + service.enable_refresh(); let task = tokio::spawn(async move { if engine_ready.wait_for(|ready| *ready).await.is_err() { return; @@ -202,6 +203,7 @@ impl PolicyDnsRuntime { store.clone(), trusted_host_gateway, )); + service.enable_refresh(); let address = udp.local_addr().into_diagnostic()?; let udp_service = service.clone(); diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index 62fe1c19bf..b663cc36e1 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -12,8 +12,8 @@ use crate::proxy::destination::{ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::ops::RangeInclusive; -use std::sync::RwLock; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; use std::time::{Duration, Instant}; use uuid::Uuid; @@ -266,7 +266,7 @@ struct Publication { struct StoreState { records: BTreeMap, allocations: BTreeMap, - expired_allocations: BTreeSet, + expired_allocations: BTreeMap, next_ipv4: u32, end_ipv4: u32, next_ipv6: u128, @@ -274,11 +274,15 @@ struct StoreState { next_mapping_generation: u64, } +type RefreshLock = Arc>>; + pub(crate) struct ResolvedEndpointStore { state: RwLock, config: StoreConfig, ipv4_pool_high_water_emitted: AtomicBool, ipv6_pool_high_water_emitted: AtomicBool, + refresher: OnceLock>, + refreshes: Mutex>, } impl ResolvedEndpointStore { @@ -291,7 +295,7 @@ impl ResolvedEndpointStore { state: RwLock::new(StoreState { records: BTreeMap::new(), allocations: BTreeMap::new(), - expired_allocations: BTreeSet::new(), + expired_allocations: BTreeMap::new(), next_ipv4, end_ipv4, next_ipv6, @@ -301,9 +305,83 @@ impl ResolvedEndpointStore { config, ipv4_pool_high_water_emitted: AtomicBool::new(false), ipv6_pool_high_water_emitted: AtomicBool::new(false), + refresher: OnceLock::new(), + refreshes: Mutex::new(BTreeMap::new()), } } + pub(super) fn set_refresher(&self, refresher: Weak) { + let _ = self.refresher.set(refresher); + } + + /// Refresh only an expired, policy-backed mapping from the same policy + /// generation. Never let an expired observation acquire an egress grant. + pub(crate) async fn refresh_if_expired( + &self, + address: IpAddr, + port: u16, + generation: u64, + ) -> miette::Result<()> { + if !matches!( + self.lookup_intent(address, port, generation, Instant::now()), + Err(MappingLookupError::Expired) + ) { + return Ok(()); + } + let record = { + let state = self + .state + .read() + .map_err(|_| miette::miette!("mapping store lock poisoned"))?; + state + .records + .get(&address) + .or_else(|| state.expired_allocations.get(&address)) + .cloned() + .ok_or_else(|| miette::miette!("expired mapping identity is unavailable"))? + }; + if record.policy_generation != generation + || record.is_observation() + || !record + .contracts + .iter() + .any(|contract| contract.port == port) + { + return Err(miette::miette!( + "expired mapping is not eligible for refresh" + )); + } + // Entries are allocated only for known mappings, bounded by the store's + // allocation budget. A failed lookup cools down for one second. + let refresh = self + .refreshes + .lock() + .map_err(|_| miette::miette!("mapping refresh lock poisoned"))? + .entry(address) + .or_default() + .clone(); + let mut failed_at = refresh.lock().await; + if self + .lookup_intent(address, port, generation, Instant::now()) + .is_ok() + { + return Ok(()); + } + if failed_at.is_some_and(|failed| failed.elapsed() < super::MIN_MAPPING_TTL) { + return Err(miette::miette!( + "transparent TCP mapping refresh recently failed" + )); + } + let refresher = self + .refresher + .get() + .and_then(Weak::upgrade) + .ok_or_else(|| miette::miette!("transparent TCP mapping refresh is unavailable"))?; + let result = refresher.refresh(&record).await; + *failed_at = result.as_ref().err().map(|_| Instant::now()); + result + } + pub(crate) fn publish( &self, request: PublishRequest, @@ -534,8 +612,9 @@ impl ResolvedEndpointStore { .filter_map(|(address, record)| (now >= record.expires_at).then_some(*address)) .collect::>(); for address in &expired { - state.records.remove(address); - state.expired_allocations.insert(*address); + if let Some(record) = state.records.remove(address) { + state.expired_allocations.insert(*address, record); + } } Ok(expired.len()) } @@ -548,7 +627,7 @@ fn live_record( now: Instant, ) -> Result<&ResolvedEndpointRecord, MappingLookupError> { let Some(record) = state.records.get(&synthetic_address) else { - return if state.expired_allocations.contains(&synthetic_address) { + return if state.expired_allocations.contains_key(&synthetic_address) { Err(MappingLookupError::Expired) } else { Err(MappingLookupError::Missing) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index a72c23c661..458b0e136d 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -781,7 +781,7 @@ async fn preauthorize_transparent_open( )); } let TransparentTarget { host, intent } = - match resolve_transparent_target(destination, policy_dns_store, opa_engine) { + match resolve_transparent_target(destination, policy_dns_store, opa_engine).await { Ok(target) => target, Err(error) => { warn!(%destination, %error, "Denied staged transparent connection"); @@ -1022,7 +1022,7 @@ struct TransparentTarget { intent: Option, } -fn resolve_transparent_target( +async fn resolve_transparent_target( destination: SocketAddr, policy_dns_store: Option<&Arc>, opa_engine: &OpaEngine, @@ -1033,6 +1033,13 @@ fn resolve_transparent_target( intent: None, }); }; + store + .refresh_if_expired( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + ) + .await?; match store.lookup_intent( destination.ip(), destination.port(), @@ -1226,6 +1233,15 @@ async fn handle_transparent_tcp_connection( let workload_addr = client.peer_addr().into_diagnostic()?; let original = original_destination(&client).into_diagnostic()?; let current_generation = opa_engine.current_generation(); + if let Err(error) = store + .refresh_if_expired(original.ip(), original.port(), current_generation) + .await + { + warn!(%original, %error, "Denied transparent connection after mapping refresh failed"); + emit_transparent_mapping_denial(workload_addr, original, None, MappingLookupError::Expired); + emit_activity(&activity_tx, true, "transparent_tcp_mapping"); + return Ok(()); + } let mapping = match store.lookup_intent( original.ip(), original.port(), @@ -2642,12 +2658,10 @@ async fn handle_mediated_connection( policy_local_transparent = true; (None, None, None) } else { - let host = resolve_transparent_target( - destination, - policy_dns_store.as_ref(), - &opa_engine, - )? - .host; + let host = + resolve_transparent_target(destination, policy_dns_store.as_ref(), &opa_engine) + .await? + .host; let (decision, connector) = transparent .authorization .map_or((None, None), |(decision, connector)| { diff --git a/docs/how-it-works/policies/network-rules.mdx b/docs/how-it-works/policies/network-rules.mdx index 119818a395..044d7aa4ec 100644 --- a/docs/how-it-works/policies/network-rules.mdx +++ b/docs/how-it-works/policies/network-rules.mdx @@ -661,11 +661,15 @@ the same connection, because OpenShell cannot check which service a non-HTTP payload addresses. OpenShell answers DNS queries for the hosts in your rules with placeholder -addresses whose TTL is at most 30 seconds. This applies to every endpoint, not +addresses whose TTL is between 1 and 30 seconds. This applies to every endpoint, not only `protocol: tcp`. The sandbox applies no DNS search domains, so request each -hostname exactly as your rules name it. Clients must resolve the hostname again -before reconnecting. A client that caches an address indefinitely can fail -after it expires, even while the endpoint remains allowed. OpenShell returns an +hostname exactly as your rules name it. When a new connection uses a cached +placeholder after its DNS mapping expires, OpenShell refreshes the destination +through its trusted resolver and validates the new addresses before connecting. +The placeholder stays the same while its endpoint policy is unchanged. A failed +refresh denies the connection; OpenShell does not use expired upstream addresses. +Policy changes can still invalidate cached mappings and require a new client DNS +lookup. OpenShell returns an empty answer to AAAA queries, so dual-stack clients must fall back to the A record. diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index ae231edb17..a0b1065b95 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -219,6 +219,7 @@ Common reason phrases emitted by the sandbox include: | `resolves to always-blocked address` | An outbound policy endpoint resolved to loopback, link-local, or unspecified. Network policy cannot authorize these destinations, even when they appear in `allowed_ips`; sandbox-local loopback connections do not use this policy path. | | `resolves to which is not in allowed_ips, connection rejected` | The destination resolved to an IP outside the policy's `allowed_ips` allowlist. | | `DNS resolution failed for :` | The proxy could not resolve the destination. | +| `transparent_tcp_mapping_denied` | A transparent connection could not use its synthetic DNS mapping, including an expired mapping whose refresh failed. Check the supervisor diagnostic for the cause; adding an allow rule does not repair a failed DNS refresh. | | `port is a blocked control-plane port, connection rejected` | The destination port matches a control-plane port (etcd, Kubernetes API, kubelet) and is always blocked. | | `request-target contains an encoded '/' (%2F)` | The L7 HTTP parser rejected an encoded slash. Configure `allow_encoded_slash: true` on a REST endpoint when the upstream requires encoded slashes. | | `l7 deny` | An L7 policy rule denied the request. | diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index d3e1c0843b..0c73ba75ba 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -472,7 +472,7 @@ serve(fixture_listener) .expect("create local-container sandbox"); let script = format!( - r#"import os, socket + r#"import os, socket, time for key in ('ALL_PROXY', 'HTTP_PROXY', 'HTTPS_PROXY', 'all_proxy', 'http_proxy', 'https_proxy'): os.environ.pop(key, None) try: @@ -486,6 +486,15 @@ assert any(ip.startswith('198.18.') or ip.startswith('198.19.') for ip in synthe with socket.create_connection(({host:?}, {port}), timeout=10) as conn: conn.sendall(b'probe') assert conn.recv(1024) == b'native-tcp-ok:probe' +# Reconnect using only the cached numeric address after the maximum mapping +# TTL. Do not perform a client DNS lookup that could republish the mapping. +cached = next(ip for ip in synthetic if ip.startswith(('198.18.', '198.19.'))) +time.sleep(31) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as conn: + conn.settimeout(10) + conn.connect((cached, {port})) + conn.sendall(b'expired-cache-probe') + assert conn.recv(1024) == b'native-tcp-ok:expired-cache-probe' mapped = next('::ffff:' + ip for ip in synthetic if ip.startswith(('198.18.', '198.19.'))) with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as conn: conn.settimeout(10) From 97e8a42fc96be51179c31eb19ed2691bc2cd2100 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 17 Nov 2026 11:14:48 -0800 Subject: [PATCH 2/4] fix(network): revalidate DNS pins and preserve refresh eligibility Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/README.md | 4 + .../src/policy_dns/mod.rs | 104 +++++++++- .../src/policy_dns/store.rs | 17 +- .../openshell-supervisor-network/src/proxy.rs | 184 +++++++++++++++++- 4 files changed, 295 insertions(+), 14 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 9088bd16a6..cc35b4dc18 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -40,6 +40,10 @@ queries. Refreshes for one address share a lock; failed refreshes back off for one second and never authorize expired upstream addresses. The service is held weakly by the store to avoid an ownership cycle. Observation-only mappings, wrong ports, and mappings from a previous policy generation cannot refresh. +Refresh eligibility uses the authored ports, independently of which ports the +latest DNS addresses satisfied. Dispatch reauthorizes mapped connections and +rebuilds their connectors from current pins instead of retaining staged pins +across a refresh. ## Reserved destinations diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index 27e5ae6163..92d1ec0217 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -24,7 +24,10 @@ mod store; mod wire; pub(crate) use name::NormalizedName; -pub(crate) use resolver::{AddressFamily, SocketTrustedResolver, TrustedAnswer, TrustedResolver}; +pub(crate) use resolver::{ + AddressFamily, ResolveError as TrustedResolveError, SocketTrustedResolver, TrustedAnswer, + TrustedResolver, +}; pub(crate) use runtime::{PolicyDnsRuntime, PolicyDnsRuntimeConfig}; pub(crate) use store::{ MappingLookup, MappingLookupError, PolicyEndpointId, PublishError, PublishRequest, @@ -123,7 +126,7 @@ impl MappingRefresher for PolicyDnsService { impl PolicyDnsService { /// Use a weak reference because the service itself owns the mapping store. - fn enable_refresh(self: &Arc) { + pub(crate) fn enable_refresh(self: &Arc) { let refresher: Arc = self.clone(); self.store.set_refresher(Arc::downgrade(&refresher)); } @@ -231,6 +234,10 @@ impl PolicyDnsService { // usable when the trusted resolver takes longer than the TTL floor. let now = now + started.elapsed(); let allocation_identity = allocation_identity(&eligible); + let eligible_ports = eligible + .iter() + .flat_map(|endpoint| endpoint.ports.iter().copied()) + .collect(); let mut contracts = Vec::new(); for endpoint in eligible { for port in endpoint.ports { @@ -268,6 +275,7 @@ impl PolicyDnsService { allocation_identity, policy_generation: snapshot.generation, ttl, + eligible_ports, contracts, }; let record = self.publish_guarded( @@ -832,7 +840,14 @@ process: { run_as_user: sandbox, run_as_group: sandbox } } fn refresh_service(ttl: Duration) -> Arc> { - let base = service(BASE_POLICY, vec![]); + refresh_service_with_policy(ttl, BASE_POLICY) + } + + fn refresh_service_with_policy( + ttl: Duration, + policy: &str, + ) -> Arc> { + let base = service(policy, vec![]); let service = Arc::new(PolicyDnsService::new( base.policy, RefreshResolver { @@ -909,6 +924,89 @@ process: { run_as_user: sandbox, run_as_group: sandbox } } } + #[tokio::test] + async fn expired_partial_mapping_recovers_every_policy_eligible_port() { + let policy = BASE_POLICY.replace( + "- { host: db.example, port: 5432, protocol: tcp }", + "- { host: db.example, port: 5432, protocol: tcp, allowed_ips: [8.8.8.8/32] }\n - { host: db.example, port: 443, protocol: tcp, allowed_ips: [8.8.4.4/32] }", + ); + for initially_partial in [false, true] { + let service = refresh_service_with_policy(Duration::ZERO, &policy); + if !initially_partial { + service + .resolver + .answer + .lock() + .unwrap() + .addresses + .push("8.8.4.4".parse().unwrap()); + } + let answer = expired_answer(&service).await; + if !initially_partial { + service.resolver.answer.lock().unwrap().addresses = + vec!["8.8.8.8".parse().unwrap()]; + service + .store + .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .await + .unwrap(); + assert!(matches!( + service.store.lookup( + answer.address, + 443, + answer.policy_generation, + Instant::now() + ), + Err(MappingLookupError::PortMismatch) + )); + tokio::time::sleep(Duration::from_millis(1100)).await; + } + // Sweeping the expired record must preserve policy eligibility, + // including ports missing from the last successful DNS contracts. + assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); + service + .resolver + .answer + .lock() + .unwrap() + .addresses + .push("8.8.4.4".parse().unwrap()); + let calls = service.resolver.calls.load(Ordering::SeqCst); + assert!( + service + .store + .refresh_if_expired(answer.address, 8443, answer.policy_generation) + .await + .is_err() + ); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), calls); + service + .store + .refresh_if_expired(answer.address, 443, answer.policy_generation) + .await + .unwrap(); + let recovered = service + .store + .lookup( + answer.address, + 443, + answer.policy_generation, + Instant::now(), + ) + .unwrap(); + assert_eq!( + recovered + .pinned_addresses_for(&[PolicyEndpointId { + policy_name: "database".into(), + endpoint_index: 1 + }]) + .unwrap(), + ["8.8.4.4".parse::().unwrap()] + ); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), calls + 1); + } + } + #[tokio::test] async fn expired_mapping_refresh_failures_deny_and_back_off_then_recover() { let service = refresh_service(Duration::ZERO); diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index b663cc36e1..2130bb2b5b 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -46,6 +46,8 @@ pub(crate) struct PublishRequest { pub(crate) allocation_identity: [u8; 32], pub(crate) policy_generation: u64, pub(crate) ttl: Duration, + /// Authored ports eligible for refresh, before DNS address filtering. + pub(crate) eligible_ports: BTreeSet, pub(crate) contracts: Vec, } @@ -59,6 +61,7 @@ pub(crate) struct ResolvedEndpointRecord { pub(crate) mapping_id: Uuid, pub(crate) created_at: Instant, pub(crate) expires_at: Instant, + pub(crate) eligible_ports: BTreeSet, pub(crate) contracts: Vec, } @@ -260,6 +263,7 @@ struct Publication { key: AllocationKey, policy_generation: u64, ttl: Duration, + eligible_ports: BTreeSet, contracts: Vec, } @@ -342,10 +346,7 @@ impl ResolvedEndpointStore { }; if record.policy_generation != generation || record.is_observation() - || !record - .contracts - .iter() - .any(|contract| contract.port == port) + || !record.eligible_ports.contains(&port) { return Err(miette::miette!( "expired mapping is not eligible for refresh" @@ -392,9 +393,11 @@ impl ResolvedEndpointStore { return Err(PublishError::StalePolicy); } if request.ttl.is_zero() + || request.eligible_ports.contains(&0) || request.contracts.is_empty() || request.contracts.iter().any(|contract| { contract.port == 0 + || !request.eligible_ports.contains(&contract.port) || contract.pinned_addresses.is_empty() || contract .pinned_addresses @@ -414,6 +417,7 @@ impl ResolvedEndpointStore { }, policy_generation: request.policy_generation, ttl: request.ttl, + eligible_ports: request.eligible_ports, contracts: request.contracts, }, now, @@ -443,6 +447,7 @@ impl ResolvedEndpointStore { }, policy_generation, ttl: super::MAX_MAPPING_TTL, + eligible_ports: BTreeSet::new(), contracts: Vec::new(), }, now, @@ -468,6 +473,7 @@ impl ResolvedEndpointStore { key, policy_generation, ttl, + eligible_ports, contracts, } = publication; let family = key.family; @@ -539,6 +545,7 @@ impl ResolvedEndpointStore { mapping_id: Uuid::new_v4(), created_at: now, expires_at: now + ttl, + eligible_ports, contracts, }; state.expired_allocations.remove(&synthetic_address); @@ -701,6 +708,7 @@ mod tests { allocation_identity: [1; 32], policy_generation: generation, ttl, + eligible_ports: BTreeSet::from([5432]), contracts: vec![ResolvedPortContract { endpoint_id: PolicyEndpointId { policy_name: "database".to_string(), @@ -1118,6 +1126,7 @@ mod tests { let mut other_port = second.clone(); other_port.port = 443; other_port.pinned_addresses = vec!["203.0.113.10".parse().unwrap()]; + request.eligible_ports.insert(443); request .contracts .extend([second.clone(), unrelated.clone(), other_port]); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 458b0e136d..6839576a5c 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -2646,6 +2646,7 @@ async fn handle_mediated_connection( .and_then(EndpointObservationSender::capture); let mut policy_local_transparent = false; let mut metadata_transparent = false; + let mut mapped_transparent_destination = None; let (mut preauthorized_decision, prevalidated_connector, preauthorized_span) = if let Some(transparent) = transparent_open { let destination = transparent.destination; @@ -2658,12 +2659,20 @@ async fn handle_mediated_connection( policy_local_transparent = true; (None, None, None) } else { - let host = + let target = resolve_transparent_target(destination, policy_dns_store.as_ref(), &opa_engine) - .await? - .host; - let (decision, connector) = transparent - .authorization + .await?; + let host = target.host; + // Dispatch may refresh a lease after staging captured its + // connector. Authorize again and build fresh pins below rather + // than pairing a refreshed mapping with the staged addresses. + let authorization = if target.intent.is_some() { + mapped_transparent_destination = Some(destination); + None + } else { + transparent.authorization + }; + let (decision, connector) = authorization .map_or((None, None), |(decision, connector)| { (Some(decision), Some(connector)) }); @@ -2985,8 +2994,27 @@ async fn handle_mediated_connection( let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); if prevalidated_connector.is_none() { - match hydrate_destination_plan(&mut decision, *backend_host_gateway, *trusted_host_gateway) - { + let destination_result = + hydrate_destination_plan(&mut decision, *backend_host_gateway, *trusted_host_gateway) + .and_then(|()| { + if let Some(destination) = mapped_transparent_destination { + let plan = policy_dns_store + .as_ref() + .ok_or(MappingLookupError::Missing) + .and_then(|store| { + pinned_transparent_plan(store, destination, &decision) + }) + .map_err(|error| DestinationDenial { + kind: DestinationDenialKind::DeclaredEndpoint, + reason: format!( + "transparent destination mapping is unavailable: {error}" + ), + })?; + decision.endpoint.destination = Some(plan); + } + Ok(()) + }); + match destination_result { Ok(()) => {} Err(denial) => { if let Some(observer) = connect_endpoint_observer.as_ref() { @@ -7842,6 +7870,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } allocation_identity: [1; 32], policy_generation: generation, ttl: std::time::Duration::from_secs(30), + eligible_ports: std::collections::BTreeSet::from([port]), contracts: vec![crate::policy_dns::ResolvedPortContract { endpoint_id: PolicyEndpointId { policy_name: policy_name.to_string(), @@ -8121,6 +8150,146 @@ process: { run_as_user: sandbox, run_as_group: sandbox } assert_eq!(connector.addrs(), &["203.0.113.8:5432".parse().unwrap()]); } + #[tokio::test] + async fn dispatch_rebuilds_staged_connector_after_mapping_expiry() { + use crate::policy_dns::{ + AddressFamily, NormalizedName, PolicyDnsService, TrustedAnswer, TrustedResolver, + }; + use std::time::{Duration, Instant}; + + struct DispatchResolver(IpAddr); + impl TrustedResolver for DispatchResolver { + async fn resolve( + &self, + _: &NormalizedName, + _: AddressFamily, + ) -> std::result::Result + { + Ok(TrustedAnswer { + addresses: vec![self.0], + ttl: Duration::ZERO, + }) + } + } + + let Some(fresh_ip) = non_loopback_test_ipv4() else { + return; + }; + // The staged connector targets A; dispatch must dial the refreshed B. + // Both listeners use the same port so only the selected IP differs. + let old_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let old_address = old_listener.local_addr().unwrap(); + let port = old_address.port(); + let fresh_listener = TcpListener::bind((fresh_ip, port)).await.unwrap(); + let policy = POLICY_DNS_OPEN_POLICY.replace( + "port: 5432, protocol: tcp", + &format!("port: {port}, protocol: tcp, tls: skip"), + ); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &policy_with_native_executable_paths(&policy), + ) + .unwrap(), + ); + let cache = Arc::new(BinaryIdentityCache::new()); + let store = policy_dns_test_store(); + let service = Arc::new(PolicyDnsService::new( + engine.clone(), + DispatchResolver(fresh_ip.into()), + store.clone(), + None, + )); + service.enable_refresh(); + let answer = service + .answer_query("db.example", AddressFamily::Ipv4, Instant::now()) + .await + .unwrap(); + let destination = SocketAddr::new(answer.address, port); + let (open, _) = staged_curl_open(destination, engine.current_generation()); + let identity = open.binary_identity; + let mut decision = authorize_supplied_identity( + &engine, + &cache, + EgressIntent::connect("db.example".into(), port), + &identity, + ); + assert!(!matches!(decision.action, NetworkAction::Deny { .. })); + decision.endpoint.destination = Some(DestinationValidationPlan { + address_authorization: destination::AddressAuthorization::PinnedResolved(vec![ + old_address.ip(), + ]), + }); + let old_connector = + destination::UpstreamConnector::new("db.example", port, vec![old_address]); + tokio::time::sleep(Duration::from_millis(1100)).await; + let (stream, mut workload) = tokio::io::duplex(4096); + let handler = tokio::spawn(handle_mediated_connection( + tokio::io::BufReader::new(Box::new(stream)), + Some(identity), + None, + Some(TransparentOpen { + destination, + authorization: Some((decision, old_connector)), + connect_span: None, + }), + Some(store.clone()), + engine, + cache, + Arc::new(AtomicU32::new(0)), + None, + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + Arc::new(None), + None, + None, + None, + None, + None, + None, + )); + let server = async { + let mut upstream = tokio::select! { + old = old_listener.accept() => panic!("dispatch dialed stale connector: {old:?}"), + fresh = fresh_listener.accept() => fresh.unwrap().0, + }; + set_tcp_nodelay_best_effort(&upstream); + let mut probe = [0; 5]; + upstream.read_exact(&mut probe).await.unwrap(); + assert_eq!(&probe, b"hello"); + upstream.write_all(b"fresh").await.unwrap(); + upstream.shutdown().await.unwrap(); + }; + let client = async { + workload.write_all(b"hello").await.unwrap(); + workload.shutdown().await.unwrap(); + let mut response = Vec::new(); + workload.read_to_end(&mut response).await.unwrap(); + assert_eq!(response, b"fresh"); + }; + tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(server, client); + handler.await.unwrap().unwrap(); + }) + .await + .expect("dispatch did not complete"); + let fresh = store + .lookup( + answer.address, + port, + answer.policy_generation, + Instant::now(), + ) + .unwrap(); + assert_eq!( + fresh.record.contracts[0].pinned_addresses, + [IpAddr::from(fresh_ip)] + ); + assert_ne!(fresh.record.mapping_id, answer.mapping_id); + } + #[tokio::test] async fn staged_transparent_pins_preserve_process_destination_authorization() { for (family, private, public, second_public) in [ @@ -8221,6 +8390,7 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} allocation_identity: [1; 32], policy_generation: engine.current_generation(), ttl: std::time::Duration::from_secs(30), + eligible_ports: std::collections::BTreeSet::from([5432]), contracts, }, engine.current_generation(), From 3b684da721e3388f4158becd639af31cb46bf0b6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 17 Nov 2026 16:18:11 -0800 Subject: [PATCH 3/4] refactor(network): centralize policy DNS connect lookups Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/README.md | 7 +- .../src/policy_dns/mod.rs | 480 +++++++++--------- .../src/policy_dns/runtime.rs | 18 +- .../src/policy_dns/store.rs | 87 +--- .../openshell-supervisor-network/src/proxy.rs | 216 ++++---- .../src/proxy/tests/ipv4_mapped.rs | 19 +- .../openshell-supervisor-network/src/run.rs | 4 +- 7 files changed, 404 insertions(+), 427 deletions(-) diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index cc35b4dc18..5bed7184b8 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -37,9 +37,10 @@ Synthetic identities remain allocated after their DNS mappings expire. New transparent connections lazily refresh expired, policy-backed mappings through the same trusted resolver and destination-validation path as sandbox DNS queries. Refreshes for one address share a lock; failed refreshes back off for -one second and never authorize expired upstream addresses. The service is held -weakly by the store to avoid an ownership cycle. Observation-only mappings, -wrong ports, and mappings from a previous policy generation cannot refresh. +one second and never authorize expired upstream addresses. +`PolicyDnsService::lookup_for_connect` owns refresh coordination and returns +the resulting mapping; the underlying store stays synchronous. Observation-only +mappings, wrong ports, and mappings from a previous policy generation cannot refresh. Refresh eligibility uses the authored ports, independently of which ports the latest DNS addresses satisfied. Dispatch reauthorizes mapped connections and rebuilds their connectors from current pins instead of retaining staged pins diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index 92d1ec0217..eb2c7375d1 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -24,10 +24,7 @@ mod store; mod wire; pub(crate) use name::NormalizedName; -pub(crate) use resolver::{ - AddressFamily, ResolveError as TrustedResolveError, SocketTrustedResolver, TrustedAnswer, - TrustedResolver, -}; +pub(crate) use resolver::{AddressFamily, SocketTrustedResolver, TrustedAnswer, TrustedResolver}; pub(crate) use runtime::{PolicyDnsRuntime, PolicyDnsRuntimeConfig}; pub(crate) use store::{ MappingLookup, MappingLookupError, PolicyEndpointId, PublishError, PublishRequest, @@ -47,7 +44,9 @@ use openshell_ocsf::{ NetworkActivityBuilder, SeverityId, StateId, StatusId, }; use sha2::{Digest, Sha256}; -use std::sync::Arc; +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; pub(crate) const MIN_MAPPING_TTL: Duration = Duration::from_secs(1); @@ -58,7 +57,7 @@ pub(crate) const POLICY_LOCAL_ADDRESS: std::net::Ipv4Addr = std::net::Ipv4Addr:: #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct SyntheticAnswer { - pub(crate) address: std::net::IpAddr, + pub(crate) address: IpAddr, pub(crate) ttl: Duration, pub(crate) mapping_id: uuid::Uuid, pub(crate) mapping_generation: u64, @@ -89,47 +88,24 @@ pub(crate) enum PolicyDnsError { /// /// No socket is bound by this type. A later runtime adapter owns listener and /// namespace lifecycle and calls the bounded wire helpers in this module. -pub(crate) struct PolicyDnsService { +pub(crate) struct PolicyDnsService { policy: Arc, resolver: R, store: Arc, - trusted_host_gateway: Option, -} - -#[async_trait::async_trait] -pub(super) trait MappingRefresher: Send + Sync { - async fn refresh(&self, record: &ResolvedEndpointRecord) -> miette::Result<()>; + trusted_host_gateway: Option, + refreshes: Mutex>, } -#[async_trait::async_trait] -impl MappingRefresher for PolicyDnsService { - async fn refresh(&self, record: &ResolvedEndpointRecord) -> miette::Result<()> { - if self.policy.current_generation() != record.policy_generation { - return Err(miette::miette!("policy changed before mapping refresh")); - } - let answer = self - .answer_query( - record.normalized_name.as_str(), - record.family, - Instant::now(), - ) - .await - .map_err(|error| miette::miette!("transparent TCP mapping refresh failed: {error}"))?; - if (answer.address, answer.policy_generation) - != (record.synthetic_address, record.policy_generation) - { - return Err(miette::miette!("policy changed during mapping refresh")); - } - Ok(()) - } -} +type RefreshLock = Arc>>; -impl PolicyDnsService { - /// Use a weak reference because the service itself owns the mapping store. - pub(crate) fn enable_refresh(self: &Arc) { - let refresher: Arc = self.clone(); - self.store.set_refresher(Arc::downgrade(&refresher)); - } +#[derive(Debug, thiserror::Error)] +pub(crate) enum ConnectLookupError { + #[error("transparent destination mapping is unavailable: {0}")] + Mapping(#[from] MappingLookupError), + #[error("transparent TCP mapping refresh failed: {0}")] + Refresh(#[from] PolicyDnsError), + #[error("transparent TCP mapping refresh recently failed")] + Backoff, } impl PolicyDnsService { @@ -137,16 +113,69 @@ impl PolicyDnsService { policy: Arc, resolver: R, store: Arc, - trusted_host_gateway: Option, + trusted_host_gateway: Option, ) -> Self { Self { policy, resolver, store, trusted_host_gateway, + refreshes: Mutex::new(BTreeMap::new()), } } + /// Return connection intent, refreshing only expired policy-backed mappings. + /// The caller must still authorize the process and pin the current mapping. + pub(crate) async fn lookup_for_connect( + &self, + address: IpAddr, + port: u16, + ) -> Result { + let generation = self.policy.current_generation(); + let lookup = || { + self.store + .lookup_intent(address, port, generation, Instant::now()) + }; + match lookup() { + Err(MappingLookupError::Expired) => {} + result => return result.map_err(Into::into), + } + let record = self.store.expired_for_connect(address, port, generation)?; + // Only known, eligible allocations acquire entries, bounded by the + // store's allocation budget. Concurrent connects share one refresh. + let refresh = self + .refreshes + .lock() + .map_err(|_| MappingLookupError::LockPoisoned)? + .entry(address) + .or_default() + .clone(); + let mut failed_at = refresh.lock().await; + match lookup() { + Err(MappingLookupError::Expired) => {} + result => return result.map_err(Into::into), + } + if self.policy.current_generation() != generation { + return Err(MappingLookupError::StalePolicy.into()); + } + if failed_at.is_some_and(|failed| failed.elapsed() < MIN_MAPPING_TTL) { + return Err(ConnectLookupError::Backoff); + } + let answer = self + .answer_query( + record.normalized_name.as_str(), + record.family, + Instant::now(), + ) + .await; + *failed_at = answer.as_ref().err().map(|_| Instant::now()); + let answer = answer?; + if (answer.address, answer.policy_generation) != (record.synthetic_address, generation) { + return Err(MappingLookupError::StalePolicy.into()); + } + lookup().map_err(Into::into) + } + pub(crate) async fn answer_query( &self, raw_name: &str, @@ -406,7 +435,7 @@ struct EligibleEndpoint { fn eligible_endpoints( endpoints: &[crate::opa::MatchedEndpoint], name: &NormalizedName, - trusted_host_gateway: Option, + trusted_host_gateway: Option, ) -> Result, PolicyDnsError> { let mut eligible = Vec::new(); for endpoint in endpoints { @@ -812,63 +841,50 @@ landlock: { compatibility: best_effort } process: { run_as_user: sandbox, run_as_group: sandbox } "; - struct RefreshResolver { + struct ScriptedResolver { + answers: Mutex>>, calls: AtomicUsize, - answer: std::sync::Mutex, - fail: std::sync::atomic::AtomicBool, - pause: std::sync::atomic::AtomicBool, - started: Notify, - release: Notify, } - impl TrustedResolver for RefreshResolver { + impl TrustedResolver for ScriptedResolver { async fn resolve( &self, - _name: &NormalizedName, - _family: AddressFamily, + _: &NormalizedName, + _: AddressFamily, ) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); - if self.pause.load(Ordering::SeqCst) { - self.started.notify_one(); - self.release.notified().await; - } - if self.fail.load(Ordering::SeqCst) { - return Err(resolver::ResolveError::NxDomain); - } - Ok(self.answer.lock().unwrap().clone()) + self.answers + .lock() + .unwrap() + .pop_front() + .expect("unexpected DNS query") } } - fn refresh_service(ttl: Duration) -> Arc> { - refresh_service_with_policy(ttl, BASE_POLICY) + fn dns_answer(addresses: &[&str], ttl: Duration) -> TrustedAnswer { + TrustedAnswer { + addresses: addresses.iter().map(|ip| ip.parse().unwrap()).collect(), + ttl, + } } - fn refresh_service_with_policy( - ttl: Duration, + fn scripted_service( policy: &str, - ) -> Arc> { + answers: Vec>, + ) -> PolicyDnsService { let base = service(policy, vec![]); - let service = Arc::new(PolicyDnsService::new( + PolicyDnsService::new( base.policy, - RefreshResolver { + ScriptedResolver { + answers: Mutex::new(answers.into()), calls: AtomicUsize::new(0), - answer: std::sync::Mutex::new(TrustedAnswer { - addresses: vec!["8.8.8.8".parse().unwrap()], - ttl, - }), - fail: false.into(), - pause: false.into(), - started: Notify::new(), - release: Notify::new(), }, base.store, None, - )); - service.enable_refresh(); - service + ) } - async fn expired_answer(service: &PolicyDnsService) -> SyntheticAnswer { + async fn expired_answer(service: &PolicyDnsService) -> SyntheticAnswer { service .answer_query( "db.example", @@ -887,38 +903,33 @@ process: { run_as_user: sandbox, run_as_group: sandbox } Duration::from_mins(5), ] { for sweep in [false, true] { - let service = refresh_service(ttl); + let service = scripted_service( + BASE_POLICY, + vec![ + Ok(dns_answer(&["8.8.8.8"], ttl)), + Ok(dns_answer(&["8.8.4.4"], ttl)), + ], + ); let answer = expired_answer(&service).await; if sweep { assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); } - service.resolver.answer.lock().unwrap().addresses = - vec!["8.8.4.4".parse().unwrap()]; - service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) - .await - .unwrap(); let refreshed = service - .store - .lookup( - answer.address, - 5432, - answer.policy_generation, - Instant::now(), - ) + .lookup_for_connect(answer.address, 5432) + .await .unwrap(); + assert_eq!(refreshed.record.synthetic_address, answer.address); assert_eq!( refreshed.record.contracts[0].pinned_addresses, ["8.8.4.4".parse::().unwrap()] ); assert_ne!(refreshed.record.mapping_id, answer.mapping_id); assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); - service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) + let reused = service + .lookup_for_connect(answer.address, 5432) .await .unwrap(); + assert_eq!(reused.record.mapping_id, refreshed.record.mapping_id); assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); } } @@ -931,68 +942,45 @@ process: { run_as_user: sandbox, run_as_group: sandbox } "- { host: db.example, port: 5432, protocol: tcp, allowed_ips: [8.8.8.8/32] }\n - { host: db.example, port: 443, protocol: tcp, allowed_ips: [8.8.4.4/32] }", ); for initially_partial in [false, true] { - let service = refresh_service_with_policy(Duration::ZERO, &policy); + let mut answers = vec![Ok(dns_answer( + if initially_partial { + &["8.8.8.8"] + } else { + &["8.8.8.8", "8.8.4.4"] + }, + Duration::ZERO, + ))]; if !initially_partial { - service - .resolver - .answer - .lock() - .unwrap() - .addresses - .push("8.8.4.4".parse().unwrap()); + answers.push(Ok(dns_answer(&["8.8.8.8"], Duration::ZERO))); } + answers.push(Ok(dns_answer(&["8.8.8.8", "8.8.4.4"], Duration::ZERO))); + let service = scripted_service(&policy, answers); let answer = expired_answer(&service).await; if !initially_partial { - service.resolver.answer.lock().unwrap().addresses = - vec!["8.8.8.8".parse().unwrap()]; service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .lookup_for_connect(answer.address, 5432) .await .unwrap(); assert!(matches!( - service.store.lookup( - answer.address, - 443, - answer.policy_generation, - Instant::now() - ), - Err(MappingLookupError::PortMismatch) + service.lookup_for_connect(answer.address, 443).await, + Err(ConnectLookupError::Mapping( + MappingLookupError::PortMismatch + )) )); tokio::time::sleep(Duration::from_millis(1100)).await; } - // Sweeping the expired record must preserve policy eligibility, - // including ports missing from the last successful DNS contracts. assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); - service - .resolver - .answer - .lock() - .unwrap() - .addresses - .push("8.8.4.4".parse().unwrap()); let calls = service.resolver.calls.load(Ordering::SeqCst); assert!( service - .store - .refresh_if_expired(answer.address, 8443, answer.policy_generation) + .lookup_for_connect(answer.address, 8443) .await .is_err() ); assert_eq!(service.resolver.calls.load(Ordering::SeqCst), calls); - service - .store - .refresh_if_expired(answer.address, 443, answer.policy_generation) - .await - .unwrap(); let recovered = service - .store - .lookup( - answer.address, - 443, - answer.policy_generation, - Instant::now(), - ) + .lookup_for_connect(answer.address, 443) + .await .unwrap(); assert_eq!( recovered @@ -1009,18 +997,25 @@ process: { run_as_user: sandbox, run_as_group: sandbox } #[tokio::test] async fn expired_mapping_refresh_failures_deny_and_back_off_then_recover() { - let service = refresh_service(Duration::ZERO); + let service = scripted_service( + BASE_POLICY, + vec![ + Ok(dns_answer(&["8.8.8.8"], Duration::ZERO)), + Err(resolver::ResolveError::NxDomain), + Ok(dns_answer(&["8.8.4.4"], Duration::ZERO)), + ], + ); let answer = expired_answer(&service).await; - service.resolver.fail.store(true, Ordering::SeqCst); - for _ in 0..2 { - assert!( - service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) - .await - .is_err() - ); - } + assert!(matches!( + service.lookup_for_connect(answer.address, 5432).await, + Err(ConnectLookupError::Refresh(PolicyDnsError::Resolver( + resolver::ResolveError::NxDomain + ))) + )); + assert!(matches!( + service.lookup_for_connect(answer.address, 5432).await, + Err(ConnectLookupError::Backoff) + )); assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); assert!(matches!( service.store.lookup( @@ -1031,25 +1026,31 @@ process: { run_as_user: sandbox, run_as_group: sandbox } ), Err(MappingLookupError::Expired) )); - service.resolver.fail.store(false, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(1100)).await; - service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) + let recovered = service + .lookup_for_connect(answer.address, 5432) .await .unwrap(); + assert_eq!( + recovered.record.contracts[0].pinned_addresses, + ["8.8.4.4".parse::().unwrap()] + ); assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 3); } #[tokio::test] async fn expired_mapping_refresh_rejects_new_forbidden_addresses() { - let service = refresh_service(Duration::ZERO); + let service = scripted_service( + BASE_POLICY, + vec![ + Ok(dns_answer(&["8.8.8.8"], Duration::ZERO)), + Ok(dns_answer(&["127.0.0.1"], Duration::ZERO)), + ], + ); let answer = expired_answer(&service).await; - service.resolver.answer.lock().unwrap().addresses = vec!["127.0.0.1".parse().unwrap()]; assert!( service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) + .lookup_for_connect(answer.address, 5432) .await .is_err() ); @@ -1065,88 +1066,102 @@ process: { run_as_user: sandbox, run_as_group: sandbox } } #[tokio::test] - async fn expired_mapping_refresh_does_not_upgrade_ports_observations_or_policy_generations() { - let service = refresh_service(Duration::ZERO); - let answer = expired_answer(&service).await; - assert!( - service - .store - .refresh_if_expired(answer.address, 3306, answer.policy_generation) - .await - .is_err() - ); - let observed = service - .answer_query( - "unknown.example", - AddressFamily::Ipv4, - Instant::now().checked_sub(Duration::from_mins(1)).unwrap(), - ) - .await - .unwrap(); - assert!( - service - .store - .refresh_if_expired(observed.address, 5432, observed.policy_generation) - .await - .is_err() - ); - service - .policy - .reload( - include_str!("../../data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - assert!( - service - .store - .refresh_if_expired(answer.address, 5432, service.policy.current_generation()) - .await - .is_err() - ); - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 1); + async fn connect_lookup_rejects_ineligible_refresh_without_dns() { + for case in ["wrong port", "expired observation", "stale generation"] { + let service = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); + let answer = expired_answer(&service).await; + let (address, port, expected) = match case { + "wrong port" => (answer.address, 3306, MappingLookupError::PortMismatch), + "expired observation" => { + let observed = service + .answer_query( + "unknown.example", + AddressFamily::Ipv4, + Instant::now().checked_sub(Duration::from_mins(1)).unwrap(), + ) + .await + .unwrap(); + (observed.address, 5432, MappingLookupError::Expired) + } + _ => { + service + .policy + .reload( + include_str!("../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + (answer.address, 5432, MappingLookupError::StalePolicy) + } + }; + let result = service.lookup_for_connect(address, port).await; + assert!( + matches!(result, Err(ConnectLookupError::Mapping(error)) if error == expected), + "{case}: {result:?}" + ); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 1, "{case}"); + } } #[tokio::test] async fn concurrent_expired_connections_share_one_refresh() { - let service = refresh_service(Duration::ZERO); - let answer = expired_answer(&service).await; - service.resolver.pause.store(true, Ordering::SeqCst); + let base = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); + let answer = expired_answer(&base).await; + let started = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let service = Arc::new(PolicyDnsService::new( + base.policy, + BlockingResolver { + started: started.clone(), + release: release.clone(), + }, + base.store, + None, + )); let mut connects = Vec::new(); for _ in 0..16 { let service = service.clone(); connects.push(tokio::spawn(async move { - service - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) - .await + service.lookup_for_connect(answer.address, 5432).await })); } - service.resolver.started.notified().await; - // Even a resolver slower than the one-second TTL floor must publish - // a usable lease and satisfy all the waiting connections. + started.notified().await; + // Resolution takes longer than the fresh answer's one-second floor. tokio::time::sleep(Duration::from_millis(1100)).await; - service.resolver.pause.store(false, Ordering::SeqCst); - service.resolver.release.notify_one(); - for connect in connects { - connect.await.unwrap().unwrap(); - } - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), async { + let mut mapping_id = None; + for connect in connects { + let mapping = connect.await.unwrap().unwrap(); + assert_eq!( + *mapping_id.get_or_insert(mapping.record.mapping_id), + mapping.record.mapping_id + ); + } + }) + .await + .expect("one DNS response must satisfy all waiting connects"); } #[tokio::test] async fn policy_revocation_during_expired_mapping_refresh_fails_closed() { - let service = refresh_service(Duration::ZERO); - let answer = expired_answer(&service).await; - service.resolver.pause.store(true, Ordering::SeqCst); + let base = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); + let answer = expired_answer(&base).await; + let started = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let service = Arc::new(PolicyDnsService::new( + base.policy, + BlockingResolver { + started: started.clone(), + release: release.clone(), + }, + base.store, + None, + )); let refreshing = service.clone(); - let connect = tokio::spawn(async move { - refreshing - .store - .refresh_if_expired(answer.address, 5432, answer.policy_generation) - .await - }); - service.resolver.started.notified().await; + let connect = + tokio::spawn(async move { refreshing.lookup_for_connect(answer.address, 5432).await }); + started.notified().await; service .policy .reload( @@ -1154,17 +1169,12 @@ process: { run_as_user: sandbox, run_as_group: sandbox } "network_policies: {}\n", ) .unwrap(); - service.resolver.release.notify_one(); + release.notify_one(); assert!(connect.await.unwrap().is_err()); assert!( service - .store - .lookup( - answer.address, - 5432, - service.policy.current_generation(), - Instant::now() - ) + .lookup_for_connect(answer.address, 5432) + .await .is_err() ); } @@ -1292,7 +1302,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } .expect("policy-backed names keep their reserved capacity"); } - struct OcsfCapture(Arc>>); + struct OcsfCapture(Arc>>); impl tracing_subscriber::Layer for OcsfCapture { fn on_event(&self, _: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { @@ -1333,7 +1343,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } } let service = service(BASE_POLICY, Vec::new()); - let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let events = Arc::new(Mutex::new(Vec::new())); let _subscriber = tracing::subscriber::set_default( tracing_subscriber::registry().with(OcsfCapture(Arc::clone(&events))), ); @@ -1839,7 +1849,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } self.release.notified().await; Ok(TrustedAnswer { addresses: vec!["8.8.8.8".parse().unwrap()], - ttl: Duration::from_secs(10), + ttl: Duration::ZERO, }) } } diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index 9c1ce7b781..809a26f50f 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -87,7 +87,7 @@ impl PolicyDnsRuntimeConfig { } pub(crate) struct PolicyDnsRuntime { - pub(crate) store: Arc, + pub(crate) service: Arc, tasks: Vec>, } @@ -112,7 +112,7 @@ impl PolicyDnsRuntime { store.clone(), trusted_host_gateway, )); - service.enable_refresh(); + let dns_service = service.clone(); let task = tokio::spawn(async move { if engine_ready.wait_for(|ready| *ready).await.is_err() { return; @@ -128,7 +128,7 @@ impl PolicyDnsRuntime { }; let source = source.clone(); accepts.push(accept_mediated_dns(source).boxed()); - let service = service.clone(); + let service = dns_service.clone(); tokio::spawn(async move { let timing = query.timing.clone(); let response = match query.transport { @@ -162,7 +162,7 @@ impl PolicyDnsRuntime { }); } }); - let expiry_store = store.clone(); + let expiry_store = store; let expiry_task = tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); loop { @@ -179,7 +179,7 @@ impl PolicyDnsRuntime { .build() ); Ok(Self { - store, + service, tasks: vec![task, expiry_task], }) } @@ -203,7 +203,6 @@ impl PolicyDnsRuntime { store.clone(), trusted_host_gateway, )); - service.enable_refresh(); let address = udp.local_addr().into_diagnostic()?; let udp_service = service.clone(); @@ -240,6 +239,7 @@ impl PolicyDnsRuntime { }); let mut tcp_engine_ready = engine_ready; + let tcp_service = service.clone(); let tcp_task = tokio::spawn(async move { if tcp_engine_ready.wait_for(|ready| *ready).await.is_err() { return; @@ -249,7 +249,7 @@ impl PolicyDnsRuntime { break; }; set_tcp_nodelay_best_effort(&stream); - let service = service.clone(); + let service = tcp_service.clone(); tokio::spawn(async move { // DNS-over-TCP connections may carry multiple sequential // length-prefixed messages. libc commonly reuses one @@ -277,7 +277,7 @@ impl PolicyDnsRuntime { }); } }); - let expiry_store = store.clone(); + let expiry_store = store; let expiry_task = tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); loop { @@ -295,7 +295,7 @@ impl PolicyDnsRuntime { .build() ); Ok(Self { - store, + service, tasks: vec![udp_task, tcp_task, expiry_task], }) } diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index 2130bb2b5b..c38b278a6e 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -12,8 +12,8 @@ use crate::proxy::destination::{ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::ops::RangeInclusive; +use std::sync::RwLock; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; use std::time::{Duration, Instant}; use uuid::Uuid; @@ -278,15 +278,11 @@ struct StoreState { next_mapping_generation: u64, } -type RefreshLock = Arc>>; - pub(crate) struct ResolvedEndpointStore { state: RwLock, config: StoreConfig, ipv4_pool_high_water_emitted: AtomicBool, ipv6_pool_high_water_emitted: AtomicBool, - refresher: OnceLock>, - refreshes: Mutex>, } impl ResolvedEndpointStore { @@ -309,78 +305,35 @@ impl ResolvedEndpointStore { config, ipv4_pool_high_water_emitted: AtomicBool::new(false), ipv6_pool_high_water_emitted: AtomicBool::new(false), - refresher: OnceLock::new(), - refreshes: Mutex::new(BTreeMap::new()), } } - pub(super) fn set_refresher(&self, refresher: Weak) { - let _ = self.refresher.set(refresher); - } - - /// Refresh only an expired, policy-backed mapping from the same policy - /// generation. Never let an expired observation acquire an egress grant. - pub(crate) async fn refresh_if_expired( + /// Recover refresh identity without authorizing expired upstream addresses. + pub(super) fn expired_for_connect( &self, address: IpAddr, port: u16, generation: u64, - ) -> miette::Result<()> { - if !matches!( - self.lookup_intent(address, port, generation, Instant::now()), - Err(MappingLookupError::Expired) - ) { - return Ok(()); - } - let record = { - let state = self - .state - .read() - .map_err(|_| miette::miette!("mapping store lock poisoned"))?; - state - .records - .get(&address) - .or_else(|| state.expired_allocations.get(&address)) - .cloned() - .ok_or_else(|| miette::miette!("expired mapping identity is unavailable"))? - }; - if record.policy_generation != generation - || record.is_observation() - || !record.eligible_ports.contains(&port) - { - return Err(miette::miette!( - "expired mapping is not eligible for refresh" - )); + ) -> Result { + let state = self + .state + .read() + .map_err(|_| MappingLookupError::LockPoisoned)?; + let record = state + .records + .get(&address) + .or_else(|| state.expired_allocations.get(&address)) + .ok_or(MappingLookupError::Missing)?; + if record.policy_generation != generation { + return Err(MappingLookupError::StalePolicy); } - // Entries are allocated only for known mappings, bounded by the store's - // allocation budget. A failed lookup cools down for one second. - let refresh = self - .refreshes - .lock() - .map_err(|_| miette::miette!("mapping refresh lock poisoned"))? - .entry(address) - .or_default() - .clone(); - let mut failed_at = refresh.lock().await; - if self - .lookup_intent(address, port, generation, Instant::now()) - .is_ok() - { - return Ok(()); + if record.is_observation() { + return Err(MappingLookupError::Expired); } - if failed_at.is_some_and(|failed| failed.elapsed() < super::MIN_MAPPING_TTL) { - return Err(miette::miette!( - "transparent TCP mapping refresh recently failed" - )); + if !record.eligible_ports.contains(&port) { + return Err(MappingLookupError::PortMismatch); } - let refresher = self - .refresher - .get() - .and_then(Weak::upgrade) - .ok_or_else(|| miette::miette!("transparent TCP mapping refresh is unavailable"))?; - let result = refresher.refresh(&record).await; - *failed_at = result.as_ref().err().map(|_| Instant::now()); - result + Ok(record.clone()) } pub(crate) fn publish( diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6839576a5c..96da43aae1 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -17,8 +17,11 @@ use openshell_sandbox_backend::test_support as rpc_test; use crate::identity::{BinaryIdentityCache, SuppliedIdentityError}; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; +#[cfg(test)] +use crate::policy_dns::SocketTrustedResolver; use crate::policy_dns::{ - MappingLookup, MappingLookupError, PolicyEndpointId, ResolvedEndpointStore, + ConnectLookupError, MappingLookup, MappingLookupError, PolicyDnsService, PolicyEndpointId, + ResolvedEndpointStore, }; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::telemetry::egress_emit; @@ -311,7 +314,7 @@ impl ProxyHandle { upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, backend_host_gateway: Option, network_mediation_source: Option>, - policy_dns_store: Option>, + policy_dns: Option>, direct_listener_identity: Option, required_proxy_authorization: Option>, reserved_destination: Option, @@ -477,7 +480,7 @@ impl ProxyHandle { match pending { Ok(connection) => { let tx = preauthorized_tx.clone(); - let dns_store = policy_dns_store.clone(); + let dns_service = policy_dns.clone(); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let backend_gateway = *backend_host_gateway; @@ -488,7 +491,7 @@ impl ProxyHandle { tokio::spawn(async move { if let Some(connection) = preauthorize_transparent_open( connection, - dns_store.as_ref(), + dns_service.as_ref(), &opa, &cache, backend_gateway, @@ -548,7 +551,7 @@ impl ProxyHandle { let backend_gw = backend_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); let credentials = provider_credentials.clone(); - let dns_store = policy_dns_store.clone(); + let dns_service = policy_dns.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); @@ -563,7 +566,7 @@ impl ProxyHandle { supplied_identity, socket_addrs, transparent_destination, - dns_store, + dns_service, opa, cache, spid, @@ -634,7 +637,7 @@ impl ProxyHandle { #[allow(clippy::too_many_arguments)] async fn preauthorize_transparent_open( connection: PendingTcpOpen, - policy_dns_store: Option<&Arc>, + policy_dns: Option<&Arc>, opa_engine: &OpaEngine, identity_cache: &BinaryIdentityCache, backend_host_gateway: Option, @@ -781,7 +784,7 @@ async fn preauthorize_transparent_open( )); } let TransparentTarget { host, intent } = - match resolve_transparent_target(destination, policy_dns_store, opa_engine).await { + match resolve_transparent_target(destination, policy_dns).await { Ok(target) => target, Err(error) => { warn!(%destination, %error, "Denied staged transparent connection"); @@ -868,9 +871,9 @@ async fn preauthorize_transparent_open( // no addresses, so they never reach an upstream dial. let pinned_plan = match &intent { None => None, - Some(intent) => match policy_dns_store + Some(intent) => match policy_dns .ok_or(MappingLookupError::Missing) - .and_then(|store| pinned_transparent_plan(store, destination, &decision)) + .and_then(|service| pinned_transparent_plan(service.store(), destination, &decision)) { Ok(plan) => Some(plan), Err(error) => { @@ -1024,39 +1027,27 @@ struct TransparentTarget { async fn resolve_transparent_target( destination: SocketAddr, - policy_dns_store: Option<&Arc>, - opa_engine: &OpaEngine, + policy_dns: Option<&Arc>, ) -> Result { - let Some(store) = policy_dns_store else { + let Some(service) = policy_dns else { return Ok(TransparentTarget { host: destination.ip().to_string(), intent: None, }); }; - store - .refresh_if_expired( - destination.ip(), - destination.port(), - opa_engine.current_generation(), - ) - .await?; - match store.lookup_intent( - destination.ip(), - destination.port(), - opa_engine.current_generation(), - std::time::Instant::now(), - ) { + match service + .lookup_for_connect(destination.ip(), destination.port()) + .await + { Ok(mapping) => Ok(TransparentTarget { host: mapping.record.normalized_name.as_str().to_string(), intent: Some(mapping), }), - Err(MappingLookupError::Missing) => Ok(TransparentTarget { + Err(ConnectLookupError::Mapping(MappingLookupError::Missing)) => Ok(TransparentTarget { host: destination.ip().to_string(), intent: None, }), - Err(error) => Err(miette::miette!( - "transparent destination mapping is unavailable: {error}" - )), + Err(error) => Err(miette::miette!("{error}")), } } @@ -1131,7 +1122,7 @@ impl TransparentTcpHandle { #[allow(clippy::too_many_arguments)] pub(crate) fn start( listeners: Vec, - store: Arc, + dns: Arc, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1147,7 +1138,7 @@ impl TransparentTcpHandle { ); let mut joins = Vec::with_capacity(listeners.len()); for listener in listeners { - let store = store.clone(); + let dns = dns.clone(); let engine = opa_engine.clone(); let cache = identity_cache.clone(); let pid = entrypoint_pid.clone(); @@ -1173,7 +1164,7 @@ impl TransparentTcpHandle { break; }; set_tcp_nodelay_best_effort(&stream); - let store = store.clone(); + let dns = dns.clone(); let engine = engine.clone(); let cache = cache.clone(); let pid = pid.clone(); @@ -1184,7 +1175,7 @@ impl TransparentTcpHandle { tokio::spawn(async move { if let Err(error) = handle_transparent_tcp_connection( stream, - store, + dns, engine, cache, pid, @@ -1221,7 +1212,7 @@ impl Drop for TransparentTcpHandle { #[allow(clippy::too_many_arguments)] async fn handle_transparent_tcp_connection( mut client: TcpStream, - store: Arc, + dns: Arc, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1232,25 +1223,17 @@ async fn handle_transparent_tcp_connection( ) -> Result<()> { let workload_addr = client.peer_addr().into_diagnostic()?; let original = original_destination(&client).into_diagnostic()?; - let current_generation = opa_engine.current_generation(); - if let Err(error) = store - .refresh_if_expired(original.ip(), original.port(), current_generation) - .await - { - warn!(%original, %error, "Denied transparent connection after mapping refresh failed"); - emit_transparent_mapping_denial(workload_addr, original, None, MappingLookupError::Expired); - emit_activity(&activity_tx, true, "transparent_tcp_mapping"); - return Ok(()); - } - let mapping = match store.lookup_intent( - original.ip(), - original.port(), - current_generation, - std::time::Instant::now(), - ) { + let mapping = match dns.lookup_for_connect(original.ip(), original.port()).await { Ok(mapping) => mapping, Err(error) => { - emit_transparent_mapping_denial(workload_addr, original, None, error); + warn!(%original, %error, "Denied transparent connection mapping"); + let cause = match error { + ConnectLookupError::Mapping(error) => error, + ConnectLookupError::Refresh(_) | ConnectLookupError::Backoff => { + MappingLookupError::Expired + } + }; + emit_transparent_mapping_denial(workload_addr, original, None, cause); emit_activity(&activity_tx, true, "transparent_tcp_mapping"); return Ok(()); } @@ -1316,7 +1299,7 @@ async fn handle_transparent_tcp_connection( emit_activity(&activity_tx, true, "transparent_tcp_mapping"); return Ok(()); }; - let mapping = match store.lookup( + let mapping = match dns.store().lookup( original.ip(), original.port(), decision.policy_generation, @@ -2622,7 +2605,7 @@ async fn handle_mediated_connection( supplied_identity: Option>, socket_addrs: Option<(SocketAddr, SocketAddr)>, transparent_open: Option, - policy_dns_store: Option>, + policy_dns: Option>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -2659,9 +2642,7 @@ async fn handle_mediated_connection( policy_local_transparent = true; (None, None, None) } else { - let target = - resolve_transparent_target(destination, policy_dns_store.as_ref(), &opa_engine) - .await?; + let target = resolve_transparent_target(destination, policy_dns.as_ref()).await?; let host = target.host; // Dispatch may refresh a lease after staging captured its // connector. Authorize again and build fresh pins below rather @@ -2998,11 +2979,11 @@ async fn handle_mediated_connection( hydrate_destination_plan(&mut decision, *backend_host_gateway, *trusted_host_gateway) .and_then(|()| { if let Some(destination) = mapped_transparent_destination { - let plan = policy_dns_store + let plan = policy_dns .as_ref() .ok_or(MappingLookupError::Missing) - .and_then(|store| { - pinned_transparent_plan(store, destination, &decision) + .and_then(|service| { + pinned_transparent_plan(service.store(), destination, &decision) }) .map_err(|error| DestinationDenial { kind: DestinationDenialKind::DeclaredEndpoint, @@ -7855,6 +7836,18 @@ process: { run_as_user: sandbox, run_as_group: sandbox } )) } + fn policy_dns_test_service( + store: &Arc, + engine: &Arc, + ) -> Arc { + Arc::new(PolicyDnsService::new( + engine.clone(), + SocketTrustedResolver::new("127.0.0.1:9".parse().unwrap()), + store.clone(), + None, + )) + } + fn publish_mapping( store: &ResolvedEndpointStore, name: &str, @@ -8111,11 +8104,13 @@ process: { run_as_user: sandbox, run_as_group: sandbox } #[tokio::test] async fn staged_transparent_open_dials_only_pinned_policy_dns_addresses() { - let engine = native_identity_engine( - include_str!("../data/sandbox-policy.rego"), - &policy_with_native_executable_paths(POLICY_DNS_OPEN_POLICY), - ) - .unwrap(); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &policy_with_native_executable_paths(POLICY_DNS_OPEN_POLICY), + ) + .unwrap(), + ); let store = policy_dns_test_store(); let record = publish_mapping( &store, @@ -8131,7 +8126,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } let (_, _, _, transparent) = preauthorize_transparent_open( open, - Some(&store), + Some(&policy_dns_test_service(&store, &engine)), &engine, &BinaryIdentityCache::new(), None, @@ -8152,26 +8147,13 @@ process: { run_as_user: sandbox, run_as_group: sandbox } #[tokio::test] async fn dispatch_rebuilds_staged_connector_after_mapping_expiry() { - use crate::policy_dns::{ - AddressFamily, NormalizedName, PolicyDnsService, TrustedAnswer, TrustedResolver, + use crate::policy_dns::AddressFamily; + use hickory_proto::{ + op::{Message, OpCode}, + rr::{RData, Record, rdata::A}, }; use std::time::{Duration, Instant}; - struct DispatchResolver(IpAddr); - impl TrustedResolver for DispatchResolver { - async fn resolve( - &self, - _: &NormalizedName, - _: AddressFamily, - ) -> std::result::Result - { - Ok(TrustedAnswer { - addresses: vec![self.0], - ttl: Duration::ZERO, - }) - } - } - let Some(fresh_ip) = non_loopback_test_ipv4() else { return; }; @@ -8194,13 +8176,32 @@ process: { run_as_user: sandbox, run_as_group: sandbox } ); let cache = Arc::new(BinaryIdentityCache::new()); let store = policy_dns_test_store(); + let dns_socket = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let resolver = SocketTrustedResolver::new(dns_socket.local_addr().unwrap()); + let dns_task = tokio::spawn(async move { + for _ in 0..2 { + let mut wire = [0; 8192]; + let (length, peer) = dns_socket.recv_from(&mut wire).await.unwrap(); + let query = Message::from_vec(&wire[..length]).unwrap(); + let mut response = Message::response(query.metadata.id, OpCode::Query); + response.queries = query.queries.clone(); + response.add_answer(Record::from_rdata( + query.queries[0].name.clone(), + 0, + RData::A(A(fresh_ip)), + )); + dns_socket + .send_to(&response.to_vec().unwrap(), peer) + .await + .unwrap(); + } + }); let service = Arc::new(PolicyDnsService::new( engine.clone(), - DispatchResolver(fresh_ip.into()), + resolver, store.clone(), None, )); - service.enable_refresh(); let answer = service .answer_query("db.example", AddressFamily::Ipv4, Instant::now()) .await @@ -8233,7 +8234,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } authorization: Some((decision, old_connector)), connect_span: None, }), - Some(store.clone()), + Some(service.clone()), engine, cache, Arc::new(AtomicU32::new(0)), @@ -8288,6 +8289,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } [IpAddr::from(fresh_ip)] ); assert_ne!(fresh.record.mapping_id, answer.mapping_id); + dns_task.await.unwrap(); } #[tokio::test] @@ -8344,11 +8346,13 @@ landlock: {{ compatibility: best_effort }} process: {{ run_as_user: sandbox, run_as_group: sandbox }} " ); - let engine = native_identity_engine( - include_str!("../data/sandbox-policy.rego"), - &policy_with_native_executable_paths(&policy), - ) - .unwrap(); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &policy_with_native_executable_paths(&policy), + ) + .unwrap(), + ); let store = policy_dns_test_store(); let contract = |policy_name: &str, endpoint_index, addresses: Vec| { crate::policy_dns::ResolvedPortContract { @@ -8403,7 +8407,7 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} ); let accepted = preauthorize_transparent_open( open, - Some(&store), + Some(&policy_dns_test_service(&store, &engine)), &engine, &BinaryIdentityCache::new(), None, @@ -8449,11 +8453,13 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} #[tokio::test] async fn staged_transparent_open_proposes_a_denied_observation_hostname() { - let engine = native_identity_engine( - include_str!("../data/sandbox-policy.rego"), - &policy_with_native_executable_paths(POLICY_DNS_OPEN_POLICY), - ) - .unwrap(); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &policy_with_native_executable_paths(POLICY_DNS_OPEN_POLICY), + ) + .unwrap(), + ); let store = policy_dns_test_store(); let record = publish_observation(&store, "unknown.example", engine.current_generation()); let (open, completion) = staged_curl_open( @@ -8465,7 +8471,7 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} assert!( preauthorize_transparent_open( open, - Some(&store), + Some(&policy_dns_test_service(&store, &engine)), &engine, &BinaryIdentityCache::new(), None, @@ -8493,11 +8499,13 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} #[tokio::test] async fn staged_transparent_open_never_relays_an_observation_that_policy_allows() { - let engine = native_identity_engine( - include_str!("../data/sandbox-policy.rego"), - &policy_with_native_executable_paths(POLICY_DNS_OPEN_POLICY), - ) - .unwrap(); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &policy_with_native_executable_paths(POLICY_DNS_OPEN_POLICY), + ) + .unwrap(), + ); let identity_cache = BinaryIdentityCache::new(); let (denial_tx, mut denial_rx) = mpsc::unbounded_channel(); let preauthorize = |store: Arc, destination: SocketAddr| { @@ -8508,7 +8516,7 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} async move { let admitted = preauthorize_transparent_open( open, - Some(&store), + Some(&policy_dns_test_service(&store, engine)), engine, identity_cache, None, diff --git a/crates/openshell-supervisor-network/src/proxy/tests/ipv4_mapped.rs b/crates/openshell-supervisor-network/src/proxy/tests/ipv4_mapped.rs index 81bf372cb7..8b48e3d455 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/ipv4_mapped.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/ipv4_mapped.rs @@ -5,18 +5,22 @@ use super::*; -fn engine_for(host: &str) -> OpaEngine { - native_identity_engine( - include_str!("../../../data/sandbox-policy.rego"), - &policy_with_native_executable_paths(&POLICY_DNS_OPEN_POLICY.replace("db.example", host)), +fn engine_for(host: &str) -> Arc { + Arc::new( + native_identity_engine( + include_str!("../../../data/sandbox-policy.rego"), + &policy_with_native_executable_paths( + &POLICY_DNS_OPEN_POLICY.replace("db.example", host), + ), + ) + .unwrap(), ) - .unwrap() } async fn probe( destination: SocketAddr, store: Option<&Arc>, - engine: &OpaEngine, + engine: &Arc, ) -> ( Option, TcpOpenDecision, @@ -24,9 +28,10 @@ async fn probe( ) { let (open, completion) = staged_curl_open(destination, engine.current_generation()); let (denial_tx, mut denial_rx) = mpsc::unbounded_channel(); + let dns = store.map(|store| policy_dns_test_service(store, engine)); let accepted = preauthorize_transparent_open( open, - store, + dns.as_ref(), engine, &BinaryIdentityCache::new(), None, diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 0c342c6bab..169add5c5e 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -607,7 +607,7 @@ pub async fn run_networking( network_mediation_source, mediated_policy_dns .as_ref() - .map(|runtime| runtime.store.clone()), + .map(|runtime| runtime.service.clone()), proxy_listener .as_ref() .map(|proxy| proxy.binary_identity.clone()), @@ -641,7 +641,7 @@ pub async fn run_networking( )?; let transparent = crate::proxy::TransparentTcpHandle::start( runtime.listeners, - dns.store.clone(), + dns.service.clone(), engine, cache, entrypoint_pid, From 94f67ac99e5a03c3ffda2fef18076f4843125651 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 17 Nov 2026 16:31:56 -0800 Subject: [PATCH 4/4] test(network): consolidate DNS refresh scenarios Signed-off-by: Drew Newberry --- .../src/policy_dns/mod.rs | 211 ++++++------------ 1 file changed, 72 insertions(+), 139 deletions(-) diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index eb2c7375d1..ab9c3ace26 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -861,10 +861,10 @@ process: { run_as_user: sandbox, run_as_group: sandbox } } } - fn dns_answer(addresses: &[&str], ttl: Duration) -> TrustedAnswer { + fn dns_answer(addresses: &[&str]) -> TrustedAnswer { TrustedAnswer { addresses: addresses.iter().map(|ip| ip.parse().unwrap()).collect(), - ttl, + ttl: Duration::ZERO, } } @@ -896,175 +896,108 @@ process: { run_as_user: sandbox, run_as_group: sandbox } } #[tokio::test] - async fn expired_mapping_refreshes_same_address_with_new_validated_dns_answers() { - for ttl in [ - Duration::ZERO, - Duration::from_secs(5), - Duration::from_mins(5), + async fn expired_mapping_refresh_validates_backs_off_and_recovers() { + for (case, failure, sweep) in [ + ("NXDOMAIN", Err(resolver::ResolveError::NxDomain), false), + ("forbidden address", Ok(dns_answer(&["127.0.0.1"])), true), ] { - for sweep in [false, true] { - let service = scripted_service( - BASE_POLICY, - vec![ - Ok(dns_answer(&["8.8.8.8"], ttl)), - Ok(dns_answer(&["8.8.4.4"], ttl)), - ], - ); - let answer = expired_answer(&service).await; - if sweep { - assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); - } - let refreshed = service - .lookup_for_connect(answer.address, 5432) - .await - .unwrap(); - assert_eq!(refreshed.record.synthetic_address, answer.address); - assert_eq!( - refreshed.record.contracts[0].pinned_addresses, - ["8.8.4.4".parse::().unwrap()] - ); - assert_ne!(refreshed.record.mapping_id, answer.mapping_id); - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); - let reused = service - .lookup_for_connect(answer.address, 5432) - .await - .unwrap(); - assert_eq!(reused.record.mapping_id, refreshed.record.mapping_id); - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); - } - } - } - - #[tokio::test] - async fn expired_partial_mapping_recovers_every_policy_eligible_port() { - let policy = BASE_POLICY.replace( - "- { host: db.example, port: 5432, protocol: tcp }", - "- { host: db.example, port: 5432, protocol: tcp, allowed_ips: [8.8.8.8/32] }\n - { host: db.example, port: 443, protocol: tcp, allowed_ips: [8.8.4.4/32] }", - ); - for initially_partial in [false, true] { - let mut answers = vec![Ok(dns_answer( - if initially_partial { - &["8.8.8.8"] - } else { - &["8.8.8.8", "8.8.4.4"] - }, - Duration::ZERO, - ))]; - if !initially_partial { - answers.push(Ok(dns_answer(&["8.8.8.8"], Duration::ZERO))); - } - answers.push(Ok(dns_answer(&["8.8.8.8", "8.8.4.4"], Duration::ZERO))); - let service = scripted_service(&policy, answers); + let service = scripted_service( + BASE_POLICY, + vec![ + Ok(dns_answer(&["8.8.8.8"])), + failure, + Ok(dns_answer(&["8.8.4.4"])), + ], + ); let answer = expired_answer(&service).await; - if !initially_partial { - service - .lookup_for_connect(answer.address, 5432) - .await - .unwrap(); - assert!(matches!( - service.lookup_for_connect(answer.address, 443).await, - Err(ConnectLookupError::Mapping( - MappingLookupError::PortMismatch - )) - )); - tokio::time::sleep(Duration::from_millis(1100)).await; + if sweep { + assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); } - assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); - let calls = service.resolver.calls.load(Ordering::SeqCst); assert!( - service - .lookup_for_connect(answer.address, 8443) - .await - .is_err() + matches!( + service.lookup_for_connect(answer.address, 5432).await, + Err(ConnectLookupError::Refresh(_)) + ), + "{case}" ); - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), calls); + assert!(matches!( + service.store.lookup( + answer.address, + 5432, + answer.policy_generation, + Instant::now() + ), + Err(MappingLookupError::Expired) + )); + assert!(matches!( + service.lookup_for_connect(answer.address, 5432).await, + Err(ConnectLookupError::Backoff) + )); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); + + tokio::time::sleep(Duration::from_millis(1100)).await; let recovered = service - .lookup_for_connect(answer.address, 443) + .lookup_for_connect(answer.address, 5432) .await .unwrap(); + assert_eq!(recovered.record.synthetic_address, answer.address); + assert_ne!(recovered.record.mapping_id, answer.mapping_id); assert_eq!( - recovered - .pinned_addresses_for(&[PolicyEndpointId { - policy_name: "database".into(), - endpoint_index: 1 - }]) - .unwrap(), + recovered.record.contracts[0].pinned_addresses, ["8.8.4.4".parse::().unwrap()] ); - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), calls + 1); + let reused = service + .lookup_for_connect(answer.address, 5432) + .await + .unwrap(); + assert_eq!(reused.record.mapping_id, recovered.record.mapping_id); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 3); } } #[tokio::test] - async fn expired_mapping_refresh_failures_deny_and_back_off_then_recover() { + async fn expired_partial_mapping_recovers_every_policy_eligible_port() { + let policy = BASE_POLICY.replace( + "- { host: db.example, port: 5432, protocol: tcp }", + "- { host: db.example, port: 5432, protocol: tcp, allowed_ips: [8.8.8.8/32] }\n - { host: db.example, port: 443, protocol: tcp, allowed_ips: [8.8.4.4/32] }", + ); let service = scripted_service( - BASE_POLICY, + &policy, vec![ - Ok(dns_answer(&["8.8.8.8"], Duration::ZERO)), - Err(resolver::ResolveError::NxDomain), - Ok(dns_answer(&["8.8.4.4"], Duration::ZERO)), + Ok(dns_answer(&["8.8.8.8", "8.8.4.4"])), + Ok(dns_answer(&["8.8.8.8"])), + Ok(dns_answer(&["8.8.8.8", "8.8.4.4"])), ], ); let answer = expired_answer(&service).await; + service + .lookup_for_connect(answer.address, 5432) + .await + .unwrap(); assert!(matches!( - service.lookup_for_connect(answer.address, 5432).await, - Err(ConnectLookupError::Refresh(PolicyDnsError::Resolver( - resolver::ResolveError::NxDomain - ))) - )); - assert!(matches!( - service.lookup_for_connect(answer.address, 5432).await, - Err(ConnectLookupError::Backoff) - )); - assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 2); - assert!(matches!( - service.store.lookup( - answer.address, - 5432, - answer.policy_generation, - Instant::now() - ), - Err(MappingLookupError::Expired) + service.lookup_for_connect(answer.address, 443).await, + Err(ConnectLookupError::Mapping( + MappingLookupError::PortMismatch + )) )); tokio::time::sleep(Duration::from_millis(1100)).await; + assert_eq!(service.store.expire(Instant::now()).unwrap(), 1); let recovered = service - .lookup_for_connect(answer.address, 5432) + .lookup_for_connect(answer.address, 443) .await .unwrap(); assert_eq!( - recovered.record.contracts[0].pinned_addresses, + recovered + .pinned_addresses_for(&[PolicyEndpointId { + policy_name: "database".into(), + endpoint_index: 1 + }]) + .unwrap(), ["8.8.4.4".parse::().unwrap()] ); assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 3); } - #[tokio::test] - async fn expired_mapping_refresh_rejects_new_forbidden_addresses() { - let service = scripted_service( - BASE_POLICY, - vec![ - Ok(dns_answer(&["8.8.8.8"], Duration::ZERO)), - Ok(dns_answer(&["127.0.0.1"], Duration::ZERO)), - ], - ); - let answer = expired_answer(&service).await; - assert!( - service - .lookup_for_connect(answer.address, 5432) - .await - .is_err() - ); - assert!(matches!( - service.store.lookup( - answer.address, - 5432, - answer.policy_generation, - Instant::now() - ), - Err(MappingLookupError::Expired) - )); - } - #[tokio::test] async fn connect_lookup_rejects_ineligible_refresh_without_dns() { for case in ["wrong port", "expired observation", "stale generation"] {