diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 7cd4ec016b..4195e54a7e 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -726,10 +726,17 @@ pub async fn sandbox_create( Err(status) => return Err(miette::miette!(status.to_string())), }; let response = response.into_inner(); + let application_service_urls = response.application_service_urls; let service_urls = response .service_urls .into_iter() - .map(|(service, url)| (service, service_url_for_gateway(&url, &effective_server))) + .map(|(service, url)| { + let url = application_service_urls + .get(&service) + .cloned() + .unwrap_or_else(|| service_url_for_gateway(&url, &effective_server)); + (service, url) + }) .collect::>(); let sandbox = response .sandbox @@ -1122,10 +1129,7 @@ pub async fn sandbox_create( "\u{2713}".green().bold(), ); if let Some(url) = service_urls.get("").filter(|url| !url.is_empty()) { - eprintln!( - " Access at: {}", - service_url_for_gateway(url, &effective_server) - ); + eprintln!(" Access at: {url}"); } } @@ -4376,7 +4380,7 @@ pub async fn service_expose( ); } if !response.url.is_empty() { - let url = service_url_for_gateway(&response.url, server); + let url = preferred_service_url(&response, server); println!(" URL: {}", url.cyan()); } Ok(()) @@ -4589,7 +4593,7 @@ fn print_service_endpoint_table( let url = if response.url.is_empty() { String::new() } else { - service_url_for_gateway(&response.url, gateway_endpoint) + preferred_service_url(response, gateway_endpoint) }; Some(( workspace.to_string(), @@ -4680,7 +4684,7 @@ fn service_endpoint_to_json( let url = if response.url.is_empty() { String::new() } else { - service_url_for_gateway(&response.url, gateway_endpoint) + preferred_service_url(response, gateway_endpoint) }; Some(serde_json::json!({ @@ -4690,6 +4694,8 @@ fn service_endpoint_to_json( "target_port": endpoint.target_port, "authorization_mode": service_authorization_mode_name(endpoint.authorization_mode), "url": url, + "application_url": response.application_url, + "gateway_url": service_url_for_gateway(&response.gateway_url, gateway_endpoint), })) } @@ -4704,13 +4710,16 @@ fn service_authorization_mode_name(mode: i32) -> &'static str { } } -/// Read gcloud Application Default Credentials from disk. -/// -/// Returns `(client_id, client_secret, refresh_token)`. -/// -/// Checks `GOOGLE_APPLICATION_CREDENTIALS` first; falls back to -/// `$CLOUDSDK_CONFIG/application_default_credentials.json` when set, then to -/// `~/.config/gcloud/application_default_credentials.json`. +/// Application origins are independent of the gateway transport. Only legacy +/// gateway URLs inherit a caller's external gateway port. +fn preferred_service_url(response: &ServiceEndpointResponse, gateway: &str) -> String { + if response.application_url.is_empty() { + service_url_for_gateway(&response.url, gateway) + } else { + response.application_url.clone() + } +} + fn service_url_for_gateway(service_url: &str, gateway_endpoint: &str) -> String { let (Ok(mut service_url), Ok(gateway_endpoint)) = ( url::Url::parse(service_url), @@ -7384,6 +7393,8 @@ mod tests { #[test] fn service_endpoint_json_has_raw_fields_and_normalized_url() { let response = ServiceEndpointResponse { + application_url: String::new(), + gateway_url: String::new(), endpoint: Some(ServiceEndpoint { metadata: Some(ObjectMeta { workspace: "team-a".to_string(), @@ -7409,6 +7420,8 @@ mod tests { "target_port": 8080, "authorization_mode": "bearer_passthrough", "url": "https://api.openshell.localhost:17670/", + "application_url": "", + "gateway_url": "", }) ); assert!(service_endpoint_to_json(&ServiceEndpointResponse::default(), "unused").is_none()); @@ -8168,6 +8181,20 @@ mod tests { assert!(rootfs_tar_sources_supported_for_gateway(None)); } + #[test] + fn application_origin_is_preserved_when_gateway_uses_a_port_forward() { + let response = ServiceEndpointResponse { + application_url: "https://default--demo--web.services.example.com:10443/".into(), + gateway_url: "https://default--demo--web.api.example.com:8080/".into(), + url: "https://default--demo--web.services.example.com:10443/".into(), + ..Default::default() + }; + assert_eq!( + super::preferred_service_url(&response, "https://localhost:29092"), + response.application_url + ); + } + #[test] fn service_url_for_gateway_uses_external_gateway_port() { assert_eq!( diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index 89247ba718..e10ed8ffbe 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -363,6 +363,8 @@ impl OpenShell for TestOpenShell { let name = request.name; // Return a minimal sandbox with metadata for CAS operations Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(Sandbox { metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { id: format!("sb-{name}"), diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 8e7c0b190f..33f494a5b2 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -223,6 +223,8 @@ impl OpenShell for TestOpenShell { }, ); Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls, })) @@ -287,6 +289,8 @@ impl OpenShell for TestOpenShell { }; sandbox.set_phase(SandboxPhase::Ready as i32); Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -523,6 +527,8 @@ impl OpenShell for TestOpenShell { .push(request.into_inner()); Ok(Response::new( openshell_core::proto::ServiceEndpointResponse { + application_url: String::new(), + gateway_url: String::new(), url: "https://default--sandbox.openshell.localhost:17670/".to_string(), ..Default::default() }, @@ -535,6 +541,8 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Ok(Response::new( openshell_core::proto::ServiceEndpointResponse { + application_url: String::new(), + gateway_url: String::new(), url: "https://default--sandbox.openshell.localhost:17670/".to_string(), ..Default::default() }, diff --git a/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs b/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs index 772a26e58e..131e479e5e 100644 --- a/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs +++ b/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs @@ -182,6 +182,8 @@ macro_rules! mock_gateway { async fn get_sandbox(&self, _: Request) -> Result, Status> { self.calls.lock().unwrap().lookups += 1; Ok(Response::new(proto::SandboxResponse { +application_service_urls: std::collections::HashMap::new(), +gateway_service_urls: std::collections::HashMap::new(), sandbox: Some(proto::Sandbox { metadata: Some(proto::datamodel::v1::ObjectMeta { id: "test-id".to_string(), diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 497f3ce778..968169fed0 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -172,6 +172,8 @@ impl OpenShell for TestOpenShell { let name = request.name.clone(); *self.state.last_get_name.lock().await = Some(name.clone()); Ok(Response::new(SandboxResponse { + application_service_urls: std::collections::HashMap::new(), + gateway_service_urls: std::collections::HashMap::new(), sandbox: Some(Sandbox { metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { id: "test-id".to_string(), diff --git a/crates/openshell-core/src/app_listener.rs b/crates/openshell-core/src/app_listener.rs index bece387213..66d2a738f9 100644 --- a/crates/openshell-core/src/app_listener.rs +++ b/crates/openshell-core/src/app_listener.rs @@ -4,6 +4,7 @@ //! Driver-independent application listener configuration. use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; /// Explicitly enabled application ingress. Deployment details such as TLS /// Secret names belong to the driver, not to this common configuration. @@ -12,7 +13,8 @@ use serde::{Deserialize, Serialize}; pub struct AppListenerConfig { pub enabled: bool, pub port: u16, - pub domain: String, + /// Public HTTPS origin used to derive service hostnames and URLs. + pub base_url: Option, } impl Default for AppListenerConfig { @@ -20,34 +22,171 @@ impl Default for AppListenerConfig { Self { enabled: false, port: 8443, - domain: String::new(), + base_url: None, } } } +/// Validate a listener endpoint after deriving its domain from the base URL. +fn validate_listener_endpoint(port: u16, domain: &str) -> Result<(), String> { + if port < 1024 { + return Err("supervisor.app_listener.port must be at least 1024".into()); + } + if domain.is_empty() + || domain.len() > 253 + || !domain.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }) + { + return Err("app listener domain must be a lowercase DNS name".into()); + } + Ok(()) +} + impl AppListenerConfig { pub fn validate(&self) -> Result<(), String> { if !self.enabled { return Ok(()); } - if self.port < 1024 { - return Err("supervisor.app_listener.port must be at least 1024".into()); - } - if self.domain.len() > 253 - || !self.domain.split('.').all(|label| { - !label.is_empty() - && label.len() <= 63 - && !label.starts_with('-') - && !label.ends_with('-') - && label - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - }) + let origin = self.parsed_base_url()?; + validate_listener_endpoint(self.port, origin.host_str().unwrap_or_default()) + } + + fn parsed_base_url(&self) -> Result { + let base = self.base_url.as_deref().ok_or_else(|| { + "supervisor.app_listener.base_url is required when enabled".to_string() + })?; + let parsed = url::Url::parse(base) + .map_err(|_| "supervisor.app_listener.base_url must be an HTTPS origin".to_string())?; + if parsed.scheme() != "https" + || !matches!(parsed.host(), Some(url::Host::Domain(_))) + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.path() != "/" + || parsed.query().is_some() + || parsed.fragment().is_some() + || parsed.port() == Some(0) { - return Err("supervisor.app_listener.domain must be a lowercase DNS name".into()); + return Err("supervisor.app_listener.base_url must be an HTTPS origin with a DNS hostname, without credentials, path, query, or fragment".into()); + } + validate_listener_endpoint(self.port, parsed.host_str().unwrap_or_default())?; + Ok(parsed) + } + + /// Hostname suffix derived from the configured base URL. Unconfigured or + /// invalid listener settings have no service domain. + pub fn domain(&self) -> String { + self.parsed_base_url() + .ok() + .and_then(|origin| origin.host_str().map(str::to_string)) + .unwrap_or_default() + } + + /// Canonical SNI/Host shared by URL publication, configuration and discovery. + pub fn service_hostname( + &self, + workspace: &str, + sandbox: &str, + service: &str, + discriminator: Option<&str>, + ) -> String { + let label = service_route_label(discriminator, workspace, sandbox, service); + format!("{label}.{}", self.domain()) + } + + /// Configured address only; this does not imply installation or readiness. + pub fn service_url( + &self, + workspace: &str, + sandbox: &str, + service: &str, + discriminator: Option<&str>, + ) -> Option { + if !self.enabled { + return None; + } + let mut origin = self.parsed_base_url().ok()?; + origin + .set_host(Some(&self.service_hostname( + workspace, + sandbox, + service, + discriminator, + ))) + .ok()?; + Some(origin.into()) + } +} + +/// Shared naming policy for gateway-mediated and direct application URLs. +/// The ingress implementation does not participate in the public service identity. +pub fn service_route_label( + discriminator: Option<&str>, + workspace: &str, + sandbox: &str, + service: &str, +) -> String { + discriminator.map_or_else( + || { + if service.is_empty() { + format!("{workspace}--{sandbox}") + } else { + format!("{workspace}--{sandbox}--{service}") + } + }, + |gateway| service_route_hash_label(gateway, workspace, sandbox, Some(service)), + ) +} + +/// Returns the stable, one-label DNS name for a service route. +/// +/// This is a non-security naming digest, not authentication or proof of authority. +/// Preserve the existing SHA-256 input encoding and 80-bit base32 label from #4842. +pub fn service_route_hash_label( + gateway_name: &str, + workspace: &str, + sandbox: &str, + service: Option<&str>, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(gateway_name.as_bytes()); + hasher.update([0]); + hasher.update(workspace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox.as_bytes()); + hasher.update([0]); + hasher.update(service.unwrap_or("").as_bytes()); + let digest = hasher.finalize(); + base32_lower_no_padding(&digest[..10]) +} + +fn base32_lower_no_padding(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut output = String::with_capacity(16); + let mut accumulator = 0_u32; + let mut bits = 0_u8; + for byte in bytes { + accumulator = (accumulator << 8) | u32::from(*byte); + bits += 8; + while bits >= 5 { + bits -= 5; + output.push(char::from( + ALPHABET[((accumulator >> bits) & 0x1f) as usize], + )); } - Ok(()) } + if bits > 0 { + output.push(char::from( + ALPHABET[((accumulator << (5 - bits)) & 0x1f) as usize], + )); + } + output } /// A synchronous view of installed ingress, evaluated on the sandbox instance @@ -60,6 +199,99 @@ pub type InstallationSource = std::sync::Arc< mod tests { use super::*; + #[test] + fn legacy_service_route_wire_payload_keeps_readable_routing_default() { + use prost::Message as _; + let legacy = b"\x0a\x08endpoint\x12\x03web\x18\x90\x3f\x20\x01\x28\x02"; + let route = crate::proto::SandboxServiceRoute::decode(legacy.as_slice()).unwrap(); + assert_eq!(route.endpoint_id, "endpoint"); + assert_eq!(route.name, "web"); + assert_eq!(route.target_port, 8080); + assert!(route.application_hostname.is_empty()); + } + + #[test] + fn application_urls_share_gateway_hash_labels_and_preserve_public_origin() { + let config = AppListenerConfig { + enabled: true, + base_url: Some("https://apps.example.com:10443".into()), + ..Default::default() + }; + assert_eq!( + config + .service_url("default", "my-sandbox", "web", Some("gateway-a")) + .unwrap(), + "https://enh47uqyj7bpyttz.apps.example.com:10443/" + ); + assert_eq!( + config + .service_url("default", "my-sandbox", "", Some("gateway-a")) + .unwrap(), + "https://eiud2z5apv3yeucg.apps.example.com:10443/" + ); + assert_ne!( + config.service_hostname("default", "my-sandbox", "web", Some("gateway-a")), + config.service_hostname("default", "my-sandbox", "web", Some("gateway-b")) + ); + } + + #[test] + fn base_url_defines_the_service_domain_and_public_port() { + let mut config = AppListenerConfig { + enabled: true, + base_url: Some("https://services.example.com:10443".into()), + ..Default::default() + }; + config.validate().unwrap(); + assert_eq!(config.domain(), "services.example.com"); + assert_eq!( + config + .service_url("default", "demo", "web", None) + .as_deref(), + Some("https://default--demo--web.services.example.com:10443/") + ); + config.base_url = Some("https://services.example.com:443/".into()); + assert_eq!( + config.service_url("default", "demo", "", None).as_deref(), + Some("https://default--demo.services.example.com/") + ); + for bad in [ + "http://services.example.com", + "https://127.0.0.1", + "https://[::1]", + "https://services.example.com.", + "https://user:password@services.example.com", + "https://services.example.com/path", + "https://services.example.com/?query", + "https://services.example.com/#fragment", + "https://services.example.com:0", + "not-a-url", + ] { + config.base_url = Some(bad.into()); + assert!(config.validate().is_err(), "{bad}"); + } + config.base_url = Some("https://services.example.com".into()); + config.enabled = false; + assert!(config.validate().is_ok()); + assert!(config.service_url("default", "demo", "web", None).is_none()); + } + + #[test] + fn base_url_is_the_only_authored_service_domain() { + let config: AppListenerConfig = serde_json::from_str( + r#"{"enabled":true,"port":9443,"base_url":"https://services.example.com:10443"}"#, + ) + .unwrap(); + config.validate().unwrap(); + assert_eq!(config.domain(), "services.example.com"); + assert_eq!(config.port, 9443); + for field in ["domain", "public_url"] { + let value = serde_json::json!({field: "services.example.com"}); + let error = serde_json::from_value::(value).unwrap_err(); + assert!(error.to_string().contains(field)); + } + } + #[test] fn only_explicit_enablement_requires_a_valid_endpoint() { assert!(!AppListenerConfig::default().enabled); @@ -69,7 +301,7 @@ mod tests { ..Default::default() }; assert!(config.validate().is_err()); - config.domain = "services.example.com".into(); + config.base_url = Some("https://services.example.com".into()); assert!(config.validate().is_ok()); config.port = 0; assert!(config.validate().is_err()); diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 6ffeaf1dce..b455d9e9a4 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -422,6 +422,8 @@ pub struct Config { /// Part of the programmatically-built [`Config`]; never deserialized directly. #[derive(Debug, Clone)] pub struct ServiceRoutingConfig { + /// Publish gateway URLs and forward application requests through the gateway. + pub enabled: bool, /// Base domains accepted for `sandbox--service.` routes. /// The first domain is used when the gateway prints endpoint URLs. pub base_domains: Vec, @@ -1351,6 +1353,13 @@ impl Config { .then_some(self.name.as_str()) } + /// Enable gateway-mediated application routing independently of app listeners. + #[must_use] + pub const fn with_service_routing(mut self, enabled: bool) -> Self { + self.service_routing.enabled = enabled; + self + } + /// Enable or disable plaintext HTTP routing for loopback sandbox service /// hostnames on TLS-enabled gateway listeners. #[must_use] @@ -1370,6 +1379,7 @@ impl Config { impl Default for ServiceRoutingConfig { fn default() -> Self { Self { + enabled: true, base_domains: default_service_routing_domains(), enable_loopback_service_http: default_enable_loopback_service_http(), service_hostname_format: ServiceHostnameFormat::default(), diff --git a/crates/openshell-crypto/README.md b/crates/openshell-crypto/README.md index 8d411cf80d..2af0cbd57e 100644 --- a/crates/openshell-crypto/README.md +++ b/crates/openshell-crypto/README.md @@ -250,6 +250,7 @@ not a whole-application crypto coverage claim: | DNS policy contract fingerprints (`openshell-supervisor-network::policy_dns`) | These bind cached DNS state to policy and are security-relevant. They remain outside this migration; migrating the cache identity API and its failure handling is required for exclusive backend ownership. | | Credential-storage names (`credentials`, Kubernetes Secrets, Vault) | Deterministic namespace derivation, not encryption or proof of authorization. These retain their current hashing implementation and stored names; backend coverage is not asserted for them. | | Provider-refresh advisory-lock keys | Deterministic database lock selection for coordinating refreshes, not authentication or encryption. These remain outside the selected-backend contract. | +| Service-routing hostname labels (`openshell-core::app_listener`) | Deterministic public DNS naming, not authentication or proof of authorization. The existing truncated SHA-256/base32 encoding remains unchanged and uses `sha2`; gateway and application paths share the implementation. | | Endpoint IDs and pagination fingerprints | Deterministic identification/request matching, not authentication. These remain outside the selected-backend contract. | | Debug token fingerprints | Diagnostic display only; direct hashing remains outside the contract even though the input is a credential. | | Destination-trust generation labels (`openshell-server` network trust, Podman operator staging, Kubernetes ConfigMap naming, VM staged-snapshot checks) | Deterministic identity derivation for naming and generation equality, not encryption or proof of authorization. Trust decisions use the gateway's normalization, immutable staging, exact-byte validation, and TLS verification rather than these digests. These retain `sha2` directly. | diff --git a/crates/openshell-driver-kubernetes/src/app_listener.rs b/crates/openshell-driver-kubernetes/src/app_listener.rs index 0f941d1e04..95e5bfe356 100644 --- a/crates/openshell-driver-kubernetes/src/app_listener.rs +++ b/crates/openshell-driver-kubernetes/src/app_listener.rs @@ -28,7 +28,10 @@ pub fn configure_supervisor_pod(mut pod: Pod, config: &KubernetesAppListenerConf "OPENSHELL_APP_LISTEN_ADDRESS", format!("0.0.0.0:{}", config.common.port), ), - ("OPENSHELL_APP_SERVICE_DOMAIN", config.common.domain.clone()), + ( + "OPENSHELL_APP_BASE_URL", + config.common.base_url.clone().unwrap_or_default(), + ), ("OPENSHELL_APP_TLS_CERT", format!("{TLS_PATH}/tls.crt")), ("OPENSHELL_APP_TLS_KEY", format!("{TLS_PATH}/tls.key")), ] { @@ -86,4 +89,41 @@ mod tests { }; assert_eq!(configure_supervisor_pod(pod.clone(), &config), pod); } + + #[test] + fn supervisor_receives_base_url_separately_from_its_listener_port() { + let pod = Pod { + spec: Some(k8s_openapi::api::core::v1::PodSpec { + containers: vec![k8s_openapi::api::core::v1::Container { + name: "supervisor".into(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + let config = KubernetesAppListenerConfig { + common: openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + base_url: Some("https://services.example.com:10443".into()), + }, + tls_secret_name: "app-tls".into(), + }; + config.common.validate().unwrap(); + let pod = configure_supervisor_pod(pod, &config); + let env = pod.spec.unwrap().containers.remove(0).env.unwrap(); + let value = |name| { + env.iter() + .find(|entry| entry.name == name) + .unwrap() + .value + .as_deref() + }; + assert_eq!( + value("OPENSHELL_APP_BASE_URL"), + Some("https://services.example.com:10443") + ); + assert_eq!(value("OPENSHELL_APP_LISTEN_ADDRESS"), Some("0.0.0.0:9443")); + } } diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a18b51e1de..58decf7d6a 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1178,9 +1178,9 @@ mod tests { assert!(!config.app_listener.common.enabled); assert!(config.validate().is_ok()); config.app_listener.common = openshell_core::app_listener::AppListenerConfig { + base_url: Some("https://services.example.com".into()), enabled: true, port: 9443, - domain: "services.example.com".into(), }; assert!(config.validate().is_ok()); config.app_listener.common.port = config.boundary_port; diff --git a/crates/openshell-sdk/src/client.rs b/crates/openshell-sdk/src/client.rs index da415535d9..c329cfc439 100644 --- a/crates/openshell-sdk/src/client.rs +++ b/crates/openshell-sdk/src/client.rs @@ -1588,6 +1588,8 @@ fn sandbox_from_response(sandbox: Option) -> Result fn sandbox_from_create_response(response: proto::SandboxResponse) -> Result { let mut sandbox = sandbox_from_response(response.sandbox)?; sandbox.service_urls = response.service_urls; + sandbox.application_service_urls = response.application_service_urls; + sandbox.gateway_service_urls = response.gateway_service_urls; Ok(sandbox) } diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index 571ee42b68..97c1fc7574 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -402,6 +402,10 @@ pub struct SandboxRef { /// Service URLs returned by sandbox creation, keyed by service name. The /// empty key identifies the unnamed service. Non-create reads leave this empty. pub service_urls: HashMap, + /// Configured application origins; these do not establish installation readiness. + pub application_service_urls: HashMap, + /// Gateway-routed alternatives. + pub gateway_service_urls: HashMap, pub restart_count: u32, pub next_restart_at_ms: Option, pub main_process_started_at_ms: Option, @@ -455,6 +459,8 @@ impl SandboxRef { .then_some(sandbox.host_key_fingerprint), created_from_workload_template, service_urls: HashMap::new(), + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), restart_count, next_restart_at_ms, main_process_started_at_ms, diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index f98a180eea..50314c2385 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -359,7 +359,7 @@ impl OpenShell for TestOpenShell { } else { req.name.clone() }; - let service_urls = req + let service_urls: HashMap = req .service_exposures .iter() .map(|exposure| { @@ -371,6 +371,11 @@ impl OpenShell for TestOpenShell { .collect(); *self.state.last_create.lock().await = Some(req); Ok(Response::new(proto::SandboxResponse { + application_service_urls: service_urls.clone(), + gateway_service_urls: HashMap::from([( + "web".into(), + "https://web.gateway.test:8080/".into(), + )]), sandbox: Some(sandbox_with_phase(&name, proto::SandboxPhase::Provisioning)), service_urls, })) @@ -441,6 +446,8 @@ impl OpenShell for TestOpenShell { ); *self.state.last_stop.lock().await = Some(request); Ok(Response::new(proto::SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -458,6 +465,8 @@ impl OpenShell for TestOpenShell { ); *self.state.last_start.lock().await = Some(request); Ok(Response::new(proto::SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -475,6 +484,8 @@ impl OpenShell for TestOpenShell { ); *self.state.last_suspend.lock().await = Some(request); Ok(Response::new(proto::SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -492,6 +503,8 @@ impl OpenShell for TestOpenShell { ); *self.state.last_resume.lock().await = Some(request); Ok(Response::new(proto::SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -539,6 +552,8 @@ impl OpenShell for TestOpenShell { sandbox.metadata.as_mut().unwrap().id.clone_from(id); } let mut response = Response::new(proto::SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), }); @@ -1294,6 +1309,11 @@ async fn create_sandbox_passes_spec_through() { result.service_urls.get("web").map(String::as_str), Some("https://web.example.test/") ); + assert_eq!(result.application_service_urls, result.service_urls); + assert_eq!( + result.gateway_service_urls["web"], + "https://web.gateway.test:8080/" + ); let observed = state.last_create.lock().await.clone().unwrap(); assert_eq!(observed.name, "my-box"); diff --git a/crates/openshell-server/README.md b/crates/openshell-server/README.md index 6d846227ca..1d65109bef 100644 --- a/crates/openshell-server/README.md +++ b/crates/openshell-server/README.md @@ -339,3 +339,15 @@ Cache reuse adds at most one refresh interval to collection freshness; consumers must independently honor the advertised lease expiry. Reconnect and relist on stream termination; revisions are opaque equality tokens, not resumable cursors or an ordering across gateway replicas. + +## Application URL publication + +See [Supervisor application listener](../../docs/how-it-works/gateways/configuration.mdx#supervisor-application-listener) +for public URL configuration, response fields, and replay behavior. + +Application hostname mode follows `service_hostname_format` for both access +paths. The gateway sends each canonical app hostname in `SandboxServiceRoute`; +URL responses and discovery agree with that installed name. A mode or gateway +name change changes configuration identity, withdrawing old discovery evidence +until the supervisor acknowledges the replacement. Hash inputs and encoding are +shared with gateway-mediated routing in `openshell-core::app_listener`. diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index b1e015fa72..f4b827d45b 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -329,6 +329,10 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_SERVICE_HOSTNAME_FORMAT")] service_hostname_format: Option, + /// Publish gateway service URLs and enable gateway-mediated application routing. + #[arg(long, env = "OPENSHELL_SERVICE_ROUTING_ENABLED", default_value_t = true, action = ArgAction::Set)] + service_routing_enabled: bool, + /// Enable plaintext HTTP routing for loopback sandbox service URLs. #[arg( long, @@ -651,6 +655,7 @@ fn prepare_server_config_with_drivers( ) .with_server_sans(args.server_sans.clone()) .with_service_hostname_format(args.service_hostname_format.unwrap_or_default()) + .with_service_routing(args.service_routing_enabled) .with_loopback_service_http(args.enable_loopback_service_http) .with_websocket_tunnel(args.enable_websocket_tunnel); config @@ -1009,6 +1014,25 @@ async fn run_from_args( warn!("{UNAUTHENTICATED_USERS_WARNING}"); } + if !prepared.config.service_routing.enabled && !prepared.config.supervisor_app_listener.enabled + { + warn!( + "Sandbox service routing disabled: neither gateway routing nor the app listener is enabled" + ); + } + + if prepared.config.supervisor_app_listener.enabled { + info!( + domain = %prepared.config.supervisor_app_listener.domain(), + port = prepared.config.supervisor_app_listener.port, + "App-listener-based service routing enabled" + ); + } + + if prepared.config.service_routing.enabled { + info!("Gateway-based service routing enabled"); + } + info!(bind = %prepared.config.bind_address, "Starting OpenShell server"); let result = Box::pin(run_server(prepared, tracing_log_bus, compute_drivers)).await; @@ -1598,6 +1622,11 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.service_hostname_format = Some(*format); } + if let Some(enabled) = file.service_routing_enabled + && arg_defaulted(matches, "service_routing_enabled") + { + args.service_routing_enabled = enabled; + } if let Some(enabled) = file.enable_loopback_service_http && arg_defaulted(matches, "enable_loopback_service_http") { @@ -4930,6 +4959,38 @@ enable_loopback_service_http = false }); } + #[test] + fn gateway_service_routing_defaults_on_and_honors_config_precedence() { + Environment::new() + .remove("OPENSHELL_SERVICE_ROUTING_ENABLED") + .run(|| { + let (mut args, matches) = parse_with_args(&["openshell-gateway"]); + assert!(args.service_routing_enabled); + assert!(openshell_core::Config::new(None).service_routing.enabled); + let file = + config_file_from_toml("[openshell.gateway]\nservice_routing_enabled = false\n"); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert!(!args.service_routing_enabled); + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--service-routing-enabled=true"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert!(args.service_routing_enabled); + }); + Environment::new() + .set("OPENSHELL_SERVICE_ROUTING_ENABLED", "true") + .run(|| { + let file = + config_file_from_toml("[openshell.gateway]\nservice_routing_enabled = false\n"); + let (mut args, matches) = parse_with_args(&["openshell-gateway"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert!(args.service_routing_enabled); + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--service-routing-enabled=false"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert!(!args.service_routing_enabled); + }); + } + #[test] fn server_config_preparation_applies_and_validates_hashed_hostnames() { let state = tempfile::tempdir().unwrap(); @@ -4942,11 +5003,12 @@ enable_loopback_service_http = false local_tls.path().to_str().unwrap(), ) .remove("OPENSHELL_SERVICE_HOSTNAME_FORMAT") + .remove("OPENSHELL_SERVICE_ROUTING_ENABLED") .run(|| { let config_path = state.path().join("gateway.toml"); std::fs::write( &config_path, - "[openshell]\nversion = 2\n\n[openshell.gateway]\nname = \"gateway-west\"\nservice_hostname_format = \"hashed\"\n", + "[openshell]\nversion = 2\n\n[openshell.gateway]\nname = \"gateway-west\"\nservice_hostname_format = \"hashed\"\nservice_routing_enabled = false\n", ) .unwrap(); @@ -4963,6 +5025,7 @@ enable_loopback_service_http = false let prepared = super::prepare_server_config(&mut args, &matches) .expect("server config is prepared"); + assert!(!prepared.config.service_routing.enabled); assert_eq!( prepared.config.service_routing_discriminator(), Some("gateway-west") diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index b541762081..80e4a52a2e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -183,6 +183,9 @@ pub struct GatewayFileSection { pub config_delivery_mode: Option, // ── Service routing ────────────────────────────────────────────────── + /// Publish gateway URLs and forward application requests. Defaults to true. + #[serde(default)] + pub service_routing_enabled: Option, /// Subject Alternative Names configured on the gateway server certificate. /// Wildcard DNS SANs also enable sandbox service URLs under that domain. #[serde(default)] diff --git a/crates/openshell-server/src/grpc/mutation_replay.rs b/crates/openshell-server/src/grpc/mutation_replay.rs index 6bda3799b9..c6e2098e39 100644 --- a/crates/openshell-server/src/grpc/mutation_replay.rs +++ b/crates/openshell-server/src/grpc/mutation_replay.rs @@ -129,6 +129,7 @@ pub(super) trait Mutation: Message + Default + Send + Sync + 'static { request: Request, ) -> impl Future, Status>> + Send; fn capture(response: &Response) -> Result; + fn refresh_replay(_state: &ServerState, _response: &mut Self::Output) {} fn restore( store: &Store, success: Success, @@ -320,7 +321,9 @@ async fn execute_owned( return Err(replay_unavailable()); } let success = previous.success.ok_or_else(uncertain)?; - let mut response = Response::new(M::restore(&state.store, success).await?); + let mut output = M::restore(&state.store, success).await?; + M::refresh_replay(state, &mut output); + let mut response = Response::new(output); response .metadata_mut() .insert(REPLAYED_HEADER, "true".parse().expect("static metadata")); diff --git a/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs b/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs index a88c6abb9d..beaa178d49 100644 --- a/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs +++ b/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs @@ -335,7 +335,7 @@ async fn profile_scope( } macro_rules! mutation { - ($req:ty, $resp:ty, $method:literal, $handler:path, $auth:expr, $capture:expr, $restore:expr $(, $replay_method:literal)?) => { + ($req:ty, $resp:ty, $method:literal, $handler:path, $auth:expr, $capture:expr, $restore:expr $(, refresh = $refresh:expr)? $(, replay_method = $replay_method:literal)?) => { impl Mutation for $req { type Output = $resp; const METHOD: &'static str = $method; @@ -363,12 +363,15 @@ macro_rules! mutation { async fn restore(store: &Store, success: Success) -> Result { ($restore)(store, outcome(success)?).await } + fn refresh_replay(_state: &ServerState, _response: &mut Self::Output) { + $(($refresh)(_state, _response);)? + } } }; } macro_rules! scoped_mutation { - ($req:ty, $resp:ty, $method:literal, $handler:path, $role:ident, $capture:expr, $restore:expr) => { + ($req:ty, $resp:ty, $method:literal, $handler:path, $role:ident, $capture:expr, $restore:expr $(, refresh = $refresh:expr)?) => { mutation!( $req, $resp, @@ -384,13 +387,13 @@ macro_rules! scoped_mutation { .await }, $capture, - $restore + $restore $(, refresh = $refresh)? ); }; } macro_rules! sandbox_scoped_mutation { - ($req:ty, $resp:ty, $method:literal, $handler:path, $role:ident, $capture:expr, $restore:expr $(, $replay_method:literal)?) => { + ($req:ty, $resp:ty, $method:literal, $handler:path, $role:ident, $capture:expr, $restore:expr $(, refresh = $refresh:expr)? $(, replay_method = $replay_method:literal)?) => { mutation!( $req, $resp, @@ -407,7 +410,8 @@ macro_rules! sandbox_scoped_mutation { }, $capture, $restore - $(, $replay_method)? + $(, refresh = $refresh)? + $(, replay_method = $replay_method)? ); }; } @@ -442,11 +446,13 @@ macro_rules! sandbox_mutation { return Err(replay_unavailable()); }; Ok(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(live(store, &id).await?), service_urls: HashMap::new(), }) } - $(, $replay_method)? + $(, replay_method = $replay_method)? ); }; } @@ -469,10 +475,13 @@ scoped_mutation!( return Err(replay_unavailable()); }; Ok(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(live(store, &id).await?), service_urls, }) - } + }, + refresh = sandbox::refresh_response_urls ); sandbox_mutation!( openshell_core::proto::UpdateSandboxRequest, @@ -594,10 +603,13 @@ sandbox_scoped_mutation!( }; let _: Sandbox = live(store, &sandbox_id).await?; Ok(ServiceEndpointResponse { + application_url: String::new(), + gateway_url: String::new(), endpoint: Some(reference.restore(store).await?), url, }) - } + }, + refresh = service::refresh_response_urls ); macro_rules! provider_mutation { diff --git a/crates/openshell-server/src/grpc/mutation_replay/tests.rs b/crates/openshell-server/src/grpc/mutation_replay/tests.rs index c6cabf491a..ffc885a0f8 100644 --- a/crates/openshell-server/src/grpc/mutation_replay/tests.rs +++ b/crates/openshell-server/src/grpc/mutation_replay/tests.rs @@ -7,9 +7,9 @@ use crate::grpc::test_support::{authed_request, test_server_state}; use openshell_core::proto::datamodel::v1::ObjectMeta; use openshell_core::proto::open_shell_server::OpenShell; use openshell_core::proto::{ - CreateSandboxRequest, SandboxServiceExposure, SandboxSpec, SandboxWorkloadConfig, - SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, ServiceAuthorizationMode, - WorkspaceMember, WorkspaceRole, + CreateSandboxRequest, ExposeServiceRequest, SandboxServiceExposure, SandboxSpec, + SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, + ServiceAuthorizationMode, WorkspaceMember, WorkspaceRole, }; use openshell_core::rpc_error::StatusExt; use std::collections::HashMap; @@ -93,6 +93,240 @@ fn canonical_payload_ignores_map_order_and_id_but_preserves_presence() { assert_ne!(absent, fingerprint(&template).unwrap()); } +async fn url_replay_state() -> (Arc, tempfile::TempDir) { + let directory = tempfile::tempdir().unwrap(); + let key = directory.path().join("private-key"); + std::fs::write(&key, b"test-only stable private fingerprint material").unwrap(); + let mut state = test_server_state().await; + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.gateway_jwt = Some(openshell_core::config::GatewayJwtConfig { + signing_key_path: key, + public_key_path: directory.path().join("public"), + kid_path: directory.path().join("kid"), + gateway_id: "test".into(), + ttl_secs: None, + }); + (state, directory) +} + +#[tokio::test] +async fn create_url_maps_follow_independent_routing_flags_and_replays_keep_recorded_urls() { + for gateway_enabled in [false, true] { + for application_enabled in [false, true] { + let (mut state, _directory) = url_replay_state().await; + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.service_routing.enabled = gateway_enabled; + config.supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: application_enabled, + base_url: application_enabled.then(|| "https://services.example.com:10443".into()), + ..Default::default() + }; + let request = CreateSandboxRequest { + name: "routing-matrix".into(), + spec: Some(SandboxSpec::default()), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + service_exposures: ["", "web"] + .into_iter() + .map(|service| SandboxServiceExposure { + service: service.into(), + target_port: 8080, + ..Default::default() + }) + .collect(), + request_id: uuid::Uuid::new_v4().to_string(), + ..Default::default() + }; + let original = run(&state, authed_request(request.clone())) + .await + .unwrap() + .into_inner(); + assert_eq!( + original.gateway_service_urls.len(), + if gateway_enabled { 2 } else { 0 } + ); + assert_eq!( + original.application_service_urls.len(), + if application_enabled { 2 } else { 0 } + ); + for service in ["", "web"] { + if application_enabled { + assert_eq!( + original.service_urls[service], + original.application_service_urls[service] + ); + } else if gateway_enabled { + assert_eq!( + original.service_urls[service], + original.gateway_service_urls[service] + ); + } else { + assert!(original.service_urls[service].is_empty()); + } + } + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.service_routing.enabled = !gateway_enabled; + config.supervisor_app_listener.enabled = !application_enabled; + config.supervisor_app_listener.base_url = + (!application_enabled).then(|| "https://services.example.com:10443".into()); + let replay = run(&state, authed_request(request)).await.unwrap(); + assert_eq!(replay.metadata().get("openshell-replayed").unwrap(), "true"); + assert_eq!(replay.get_ref().service_urls, original.service_urls); + assert_eq!( + replay.get_ref().gateway_service_urls.len(), + if gateway_enabled { 0 } else { 2 } + ); + assert_eq!( + replay.get_ref().application_service_urls.len(), + if application_enabled { 0 } else { 2 } + ); + } + } +} + +#[tokio::test] +async fn create_replay_preserves_recorded_urls_and_refreshes_explicit_origins() { + let (mut state, _directory) = url_replay_state().await; + let request = CreateSandboxRequest { + name: "origin-replay".into(), + spec: Some(SandboxSpec::default()), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + service_exposures: ["", "web"] + .into_iter() + .map(|service| SandboxServiceExposure { + service: service.into(), + target_port: 8080, + ..Default::default() + }) + .collect(), + request_id: uuid::Uuid::new_v4().to_string(), + ..Default::default() + }; + let original = run(&state, authed_request(request.clone())) + .await + .unwrap() + .into_inner(); + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.service_routing.base_domains = vec!["gateway.example.com".into()]; + config.bind_address.set_port(12443); + config.supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + base_url: Some("https://services.example.com:10443".into()), + }; + let replay = run(&state, authed_request(request)).await.unwrap(); + assert_eq!(replay.metadata().get("openshell-replayed").unwrap(), "true"); + assert_eq!( + original.sandbox.as_ref().unwrap().metadata, + replay.get_ref().sandbox.as_ref().unwrap().metadata + ); + assert_eq!(replay.get_ref().service_urls, original.service_urls); + for (service, label) in [ + ("", "default--origin-replay"), + ("web", "default--origin-replay--web"), + ] { + assert_eq!( + replay.get_ref().application_service_urls[service], + format!("https://{label}.services.example.com:10443/") + ); + let gateway = url::Url::parse(&replay.get_ref().gateway_service_urls[service]).unwrap(); + assert_eq!( + gateway.host_str(), + Some(format!("{label}.gateway.example.com").as_str()) + ); + assert_eq!(gateway.port(), Some(12443)); + assert_ne!( + replay.get_ref().gateway_service_urls[service], + original.gateway_service_urls[service] + ); + } +} + +#[tokio::test] +async fn expose_replay_preserves_recorded_url_when_origins_change_or_are_disabled() { + let (mut state, _directory) = url_replay_state().await; + Arc::get_mut(&mut state) + .unwrap() + .config + .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + base_url: Some("https://old.example.com:10443".into()), + }; + run( + &state, + authed_request(CreateSandboxRequest { + name: "expose-replay".into(), + spec: Some(SandboxSpec::default()), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let request = ExposeServiceRequest { + sandbox: "expose-replay".into(), + name: "web".into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + target_port: 8080, + request_id: uuid::Uuid::new_v4().to_string(), + ..Default::default() + }; + let original = run(&state, authed_request(request.clone())) + .await + .unwrap() + .into_inner(); + assert_eq!( + original.url, + "https://default--expose-replay--web.old.example.com:10443/" + ); + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.service_routing.base_domains = vec!["gateway.example.com".into()]; + config.bind_address.set_port(12443); + config.supervisor_app_listener.base_url = Some("https://new.example.com:11443".into()); + let replay = run(&state, authed_request(request.clone())).await.unwrap(); + assert_eq!(replay.metadata().get("openshell-replayed").unwrap(), "true"); + assert_eq!(replay.get_ref().endpoint, original.endpoint); + assert_eq!(replay.get_ref().url, original.url); + assert_eq!( + replay.get_ref().application_url, + "https://default--expose-replay--web.new.example.com:11443/" + ); + let gateway = url::Url::parse(&replay.get_ref().gateway_url).unwrap(); + assert_eq!( + gateway.host_str(), + Some("default--expose-replay--web.gateway.example.com") + ); + assert_eq!(gateway.port(), Some(12443)); + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.supervisor_app_listener.enabled = false; + config.supervisor_app_listener.base_url = None; + let without_application = run(&state, authed_request(request.clone())).await.unwrap(); + assert_eq!( + without_application + .metadata() + .get("openshell-replayed") + .unwrap(), + "true" + ); + assert_eq!(without_application.get_ref().url, original.url); + assert_eq!(without_application.get_ref().endpoint, original.endpoint); + assert!(without_application.get_ref().application_url.is_empty()); + assert_eq!( + without_application.get_ref().gateway_url, + replay.get_ref().gateway_url + ); + Arc::get_mut(&mut state) + .unwrap() + .config + .service_routing + .enabled = false; + let without_either_path = run(&state, authed_request(request)).await.unwrap(); + assert_eq!(without_either_path.get_ref().url, original.url); + assert_eq!(without_either_path.get_ref().endpoint, original.endpoint); + assert!(without_either_path.get_ref().gateway_url.is_empty()); + assert!(without_either_path.get_ref().application_url.is_empty()); +} + #[tokio::test] async fn create_sandbox_replay_preserves_service_urls() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 46112edaf8..859740631f 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -363,6 +363,8 @@ pub(super) async fn handle_update_sandbox( ) .await?; Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -825,6 +827,8 @@ async fn handle_create_sandbox_inner( .await?; let mut service_urls = HashMap::with_capacity(request.service_exposures.len()); + let mut application_service_urls = HashMap::new(); + let mut gateway_service_urls = HashMap::new(); for exposure in &request.service_exposures { let endpoint = match super::service::expose_service_endpoint( state, @@ -866,7 +870,14 @@ async fn handle_create_sandbox_inner( return Err(exposure_error); } }; - service_urls.insert(exposure.service.clone(), endpoint.into_inner().url); + let endpoint = endpoint.into_inner(); + if !endpoint.application_url.is_empty() { + application_service_urls.insert(exposure.service.clone(), endpoint.application_url); + } + if !endpoint.gateway_url.is_empty() { + gateway_service_urls.insert(exposure.service.clone(), endpoint.gateway_url); + } + service_urls.insert(exposure.service.clone(), endpoint.url); } info!( @@ -875,6 +886,8 @@ async fn handle_create_sandbox_inner( "CreateSandbox request completed successfully" ); Ok(Response::new(SandboxResponse { + application_service_urls, + gateway_service_urls, sandbox: Some(sandbox), service_urls, })) @@ -1053,6 +1066,8 @@ pub(super) async fn handle_get_sandbox( None }; let mut response = Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), }); @@ -1879,6 +1894,8 @@ async fn handle_stop_sandbox_inner( let sandbox = state.compute.stop_sandbox(workspace, name).await?; info!(sandbox_name = %name, "StopSandbox request completed successfully"); Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -1934,6 +1951,8 @@ async fn handle_start_sandbox_inner( .project_endpoint_status(&mut sandbox, remote_authority); info!(sandbox_name = %name, "StartSandbox request completed successfully"); Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -1975,6 +1994,8 @@ async fn handle_suspend_sandbox_inner( let sandbox = state.compute.suspend_sandbox(workspace, name).await?; info!(sandbox_name = %name, "SuspendSandbox request completed successfully"); Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -2031,6 +2052,8 @@ async fn handle_resume_sandbox_inner( .project_endpoint_status(&mut sandbox, remote_authority); info!(sandbox_name = %name, "ResumeSandbox request completed successfully"); Ok(Response::new(SandboxResponse { + application_service_urls: HashMap::new(), + gateway_service_urls: HashMap::new(), sandbox: Some(sandbox), service_urls: HashMap::new(), })) @@ -4327,6 +4350,40 @@ async fn run_exec_with_russh( // Tests // --------------------------------------------------------------------------- +/// Refresh explicit access paths while preserving the recorded service URL map. +pub(super) fn refresh_response_urls(state: &ServerState, response: &mut SandboxResponse) { + let Some(sandbox) = response.sandbox.as_ref() else { + return; + }; + response.application_service_urls.clear(); + response.gateway_service_urls.clear(); + for service in response.service_urls.keys() { + let gateway = crate::service_routing::endpoint_url( + &state.config, + sandbox.object_workspace(), + sandbox.object_name(), + service, + ) + .unwrap_or_default(); + let application = state.config.supervisor_app_listener.service_url( + sandbox.object_workspace(), + sandbox.object_name(), + service, + state.config.service_routing_discriminator(), + ); + if !gateway.is_empty() { + response + .gateway_service_urls + .insert(service.clone(), gateway); + } + if let Some(application) = application { + response + .application_service_urls + .insert(service.clone(), application); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/grpc/service.rs b/crates/openshell-server/src/grpc/service.rs index 571a3a0546..4867895987 100644 --- a/crates/openshell-server/src/grpc/service.rs +++ b/crates/openshell-server/src/grpc/service.rs @@ -227,14 +227,13 @@ pub(super) async fn expose_service_endpoint( meta.resource_version = result.resource_version; } - let url = service_routing::endpoint_url(&state.config, workspace, sandbox_name, service) - .unwrap_or_default(); - service_routing::emit_service_endpoint_config_event(&endpoint, &url, created); - - Ok(Response::new(ServiceEndpointResponse { - endpoint: Some(endpoint), - url, - })) + let response = service_endpoint_response(state, endpoint); + service_routing::emit_service_endpoint_config_event( + response.endpoint.as_ref().expect("endpoint populated"), + &response.url, + created, + ); + Ok(Response::new(response)) } pub(super) async fn handle_get_service( @@ -452,6 +451,12 @@ pub(super) fn configured_app_routes_from_endpoints( target_port: ep.target_port, authorization_mode: ep.authorization_mode, resource_version: ep.metadata.as_ref().map_or(0, |m| m.resource_version), + application_hostname: state.config.supervisor_app_listener.service_hostname( + sandbox.object_workspace(), + sandbox.object_name(), + &ep.name, + state.config.service_routing_discriminator(), + ), }) .collect(); service_routes.sort_by(|a, b| a.name.cmp(&b.name)); @@ -509,14 +514,41 @@ fn service_endpoint_response( Ok(ServiceAuthorizationMode::Unspecified | ServiceAuthorizationMode::Strip) | Err(_) => ServiceAuthorizationMode::Strip as i32, }; - let workspace = endpoint.object_workspace(); - let url = - service_routing::endpoint_url(&state.config, workspace, &endpoint.sandbox, &endpoint.name) - .unwrap_or_default(); - ServiceEndpointResponse { + let mut response = ServiceEndpointResponse { endpoint: Some(endpoint), - url, - } + ..Default::default() + }; + refresh_response_urls(state, &mut response); + response.url = if response.application_url.is_empty() { + response.gateway_url.clone() + } else { + response.application_url.clone() + }; + response +} + +/// Refresh explicit access paths without changing a replay's recorded URL. +pub(super) fn refresh_response_urls(state: &ServerState, response: &mut ServiceEndpointResponse) { + let Some(endpoint) = response.endpoint.as_ref() else { + return; + }; + response.gateway_url = service_routing::endpoint_url( + &state.config, + endpoint.object_workspace(), + &endpoint.sandbox, + &endpoint.name, + ) + .unwrap_or_default(); + response.application_url = state + .config + .supervisor_app_listener + .service_url( + endpoint.object_workspace(), + &endpoint.sandbox, + &endpoint.name, + state.config.service_routing_discriminator(), + ) + .unwrap_or_default(); } #[allow(clippy::result_large_err)] @@ -603,7 +635,7 @@ mod tests { .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { enabled: true, port: 9443, - domain: "services.example.com".into(), + base_url: Some("https://services.example.com".into()), }; seed_sandbox(&state, "limited").await; state @@ -719,6 +751,109 @@ mod tests { ); } + #[tokio::test] + async fn application_and_gateway_url_publication_are_independent() { + for gateway_enabled in [false, true] { + for application_enabled in [false, true] { + let mut state = test_server_state().await; + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.service_routing.enabled = gateway_enabled; + config.supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: application_enabled, + port: 9443, + base_url: application_enabled + .then(|| "https://services.example.com:10443".into()), + }; + seed_sandbox(&state, "demo").await; + for name in ["", "web"] { + let created = handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + workspace_scope: Some(openshell_core::proto::workspace_selector( + "default", + )), + sandbox: "demo".into(), + name: name.into(), + target_port: 8080, + ..Default::default() + }), + ) + .await + .unwrap() + .into_inner(); + assert_eq!(!created.gateway_url.is_empty(), gateway_enabled); + assert_eq!(!created.application_url.is_empty(), application_enabled); + if application_enabled { + let label = if name.is_empty() { + "default--demo" + } else { + "default--demo--web" + }; + assert_eq!( + created.application_url, + format!("https://{label}.services.example.com:10443/") + ); + assert_eq!(created.url, created.application_url); + } else { + assert_eq!(created.url, created.gateway_url); + } + let fetched = handle_get_service( + &state, + authed_request(GetServiceRequest { + workspace_scope: Some(openshell_core::proto::workspace_selector( + "default", + )), + sandbox: "demo".into(), + name: name.into(), + }), + ) + .await + .unwrap() + .into_inner(); + assert_eq!(created, fetched); + } + } + } + } + + #[tokio::test] + async fn hashed_gateway_url_keeps_the_independent_application_origin() { + let mut state = test_server_state().await; + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.name = "gateway-a".into(); + config.service_routing.service_hostname_format = ServiceHostnameFormat::Hashed; + config.service_routing.base_domains = vec!["gateway.example.com".into()]; + config.supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + base_url: Some("https://services.example.com:10443".into()), + }; + let response = service_endpoint_response( + &state, + ServiceEndpoint { + metadata: Some(ObjectMeta { + workspace: "default".into(), + ..Default::default() + }), + sandbox: "demo".into(), + name: "web".into(), + ..Default::default() + }, + ); + let gateway = url::Url::parse(&response.gateway_url).unwrap(); + let label = + service_routing::service_route_hash_label("gateway-a", "default", "demo", Some("web")); + assert_eq!( + gateway.host_str(), + Some(format!("{label}.gateway.example.com").as_str()) + ); + assert_eq!( + response.application_url, + format!("https://{label}.services.example.com:10443/") + ); + assert_eq!(response.url, response.application_url); + } + #[test] fn validates_good_endpoint_name() { validate_endpoint_name("service", "web-api", 28).unwrap(); @@ -731,9 +866,9 @@ mod tests { .expect("unshared test state") .config .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + base_url: Some("https://services.example.com".into()), enabled: true, port: 9443, - domain: "services.example.com".into(), }; seed_sandbox(&state, "first").await; seed_sandbox(&state, "second").await; diff --git a/crates/openshell-server/src/grpc/service_advertisement.rs b/crates/openshell-server/src/grpc/service_advertisement.rs index 8af735722b..b69ae07ac2 100644 --- a/crates/openshell-server/src/grpc/service_advertisement.rs +++ b/crates/openshell-server/src/grpc/service_advertisement.rs @@ -385,7 +385,7 @@ async fn accept_inner( .collect(), }; if !report.routes.is_empty() { - if report.service_domain != state.config.supervisor_app_listener.domain { + if report.service_domain != state.config.supervisor_app_listener.domain() { return Err(Status::failed_precondition( "installed listener domain is not current", )); @@ -498,7 +498,7 @@ pub(super) async fn current_sandbox( ) -> Result, Status> { if !state.config.supervisor_app_listener.enabled || record.expires_at_ms <= current_time_ms() - || record.service_domain != state.config.supervisor_app_listener.domain + || record.service_domain != state.config.supervisor_app_listener.domain() || record.routes.is_empty() { return Ok(None); @@ -1003,6 +1003,7 @@ mod tests { target_port: 8080, authorization_mode: 1, resource_version: 2, + application_hostname: String::new(), } } @@ -1195,6 +1196,17 @@ pub(super) mod test_support { use super::*; pub async fn installed_fixture() -> (Arc, OwnerGuard, SupervisorServiceAdvertisement) { + installed_fixture_for_gateway( + "gateway-a", + openshell_core::config::ServiceHostnameFormat::Readable, + ) + .await + } + + pub async fn installed_fixture_for_gateway( + gateway: &str, + format: openshell_core::config::ServiceHostnameFormat, + ) -> (Arc, OwnerGuard, SupervisorServiceAdvertisement) { use openshell_core::proto::datamodel::v1::ObjectMeta; use openshell_core::proto::{SandboxPolicy, SandboxSpec, ServiceEndpoint}; let endpoint = openshell_core::proto::compute::v1::SupervisorAppEndpoint { @@ -1220,11 +1232,14 @@ pub(super) mod test_support { .unwrap() .config .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + base_url: Some("https://services.example.com".into()), enabled: true, port: 9443, - domain: "services.example.com".into(), }; - Arc::get_mut(&mut state).unwrap().admin_role = "openshell-admin".into(); + let inner = Arc::get_mut(&mut state).unwrap(); + inner.config.name = gateway.into(); + inner.config.service_routing.service_hostname_format = format; + inner.admin_role = "openshell-admin".into(); let sandbox = Sandbox { metadata: Some(ObjectMeta { id: "sandbox".into(), diff --git a/crates/openshell-server/src/grpc/service_discovery.rs b/crates/openshell-server/src/grpc/service_discovery.rs index bd8a284f5d..a97e0965b7 100644 --- a/crates/openshell-server/src/grpc/service_discovery.rs +++ b/crates/openshell-server/src/grpc/service_discovery.rs @@ -484,12 +484,25 @@ async fn publish_installations( previous.routes }; for (bytes, route) in installation.routes.iter().zip(installed_routes) { - let label = crate::service_routing::service_route_readable_label( - workspace, - sandbox.object_name(), - &route.name, - ); - let hostname = format!("{label}.{}", installation.service_domain); + let hostname = if route.application_hostname.is_empty() { + // Legacy reports can only be current against legacy configuration. + state.config.supervisor_app_listener.service_hostname( + workspace, + sandbox.object_name(), + &route.name, + state.config.service_routing_discriminator(), + ) + } else { + route.application_hostname.clone() + }; + let label = hostname + .strip_suffix(&format!(".{}", installation.service_domain)) + .ok_or_else(|| Status::internal("installed hostname domain mismatch"))?; + if label.is_empty() || label.len() > 63 || label.contains('.') { + return Err(Status::failed_precondition( + "service hostname exceeds DNS label limit", + )); + } let version = installation_version(&installation, bytes)?; routes.push(DiscoveredServiceRoute { workspace: workspace.to_string(), @@ -623,6 +636,126 @@ mod tests { assert_eq!(state.discovery_watches.available_permits(), permits); } + #[tokio::test] + async fn gateway_routing_opt_out_preserves_installed_application_discovery() { + use super::super::service_advertisement::test_support::installed_fixture_for_gateway; + use openshell_core::config::ServiceHostnameFormat; + for format in [ + ServiceHostnameFormat::Readable, + ServiceHostnameFormat::Hashed, + ] { + let (mut state, owner, report) = + installed_fixture_for_gateway("gateway-a", format).await; + let config = &mut Arc::get_mut(&mut state).unwrap().config; + config.service_routing.enabled = false; + config.supervisor_app_listener.base_url = + Some("https://services.example.com:10443".into()); + accept(&state, &owner, &report, current_time_ms() + 10_000) + .await + .unwrap(); + let discovered = snapshot(&state, "default").await.unwrap(); + assert_eq!(discovered.routes.len(), 1); + assert_eq!( + discovered.routes[0].hostname, + report.routes[0].application_hostname + ); + let response = super::super::service::handle_get_service( + &state, + authed_request(openshell_core::proto::GetServiceRequest { + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + sandbox: "demo".into(), + name: "web".into(), + }), + ) + .await + .unwrap() + .into_inner(); + assert!(response.gateway_url.is_empty()); + assert_eq!(response.url, response.application_url); + assert_eq!( + response.url, + format!("https://{}:10443/", discovered.routes[0].hostname) + ); + } + } + + #[tokio::test] + async fn hashed_urls_discovery_and_installed_routes_use_the_same_gateway_identity() { + use super::super::service_advertisement::test_support::installed_fixture_for_gateway; + use openshell_core::config::ServiceHostnameFormat; + let (mut state, owner, report) = + installed_fixture_for_gateway("gateway-a", ServiceHostnameFormat::Hashed).await; + Arc::get_mut(&mut state) + .unwrap() + .config + .supervisor_app_listener + .base_url = Some("https://services.example.com:10443".into()); + accept(&state, &owner, &report, current_time_ms() + 10_000) + .await + .unwrap(); + let first = snapshot(&state, "default").await.unwrap(); + let hostname = &first.routes[0].hostname; + let label = crate::service_routing::service_route_hash_label( + "gateway-a", + "default", + "demo", + Some("web"), + ); + assert_eq!(*hostname, format!("{label}.services.example.com")); + assert_eq!(*hostname, report.routes[0].application_hostname); + let response = super::super::service::handle_get_service( + &state, + authed_request(openshell_core::proto::GetServiceRequest { + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + sandbox: "demo".into(), + name: "web".into(), + }), + ) + .await + .unwrap() + .into_inner(); + assert_eq!( + response.application_url, + format!("https://{hostname}:10443/") + ); + assert_eq!(response.url, response.application_url); + + // A renamed installation cannot advertise a hostname it has not installed. + Arc::get_mut(&mut state).unwrap().config.name = "gateway-b".into(); + assert!(snapshot(&state, "default").await.unwrap().routes.is_empty()); + assert_eq!( + accept(&state, &owner, &report, current_time_ms() + 10_000) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + let sandbox = state + .store + .get_message::("sandbox") + .await + .unwrap() + .unwrap(); + let config = super::super::policy::build_sandbox_config_snapshot(&state, &sandbox) + .await + .unwrap(); + let replacement = openshell_core::proto::SupervisorServiceAdvertisement { + config_revision: config.config_revision, + routes: config.service_routes, + lease_sequence: report.lease_sequence + 1, + ..report + }; + accept(&state, &owner, &replacement, current_time_ms() + 10_000) + .await + .unwrap(); + let second = snapshot(&state, "default").await.unwrap(); + assert_ne!(first.routes[0].hostname, second.routes[0].hostname); + assert_ne!( + first.routes[0].installation_version, + second.routes[0].installation_version + ); + } + #[tokio::test] async fn snapshot_is_shared_between_gateway_states_and_rejects_expired_or_replaced_owners() { let (state, owner, report) = installed_fixture().await; diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 4f591f004f..182fdcee67 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -3126,11 +3126,22 @@ mod tests { async fn start_plaintext_gateway_listener() -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { + start_plaintext_gateway_listener_with_service_routing(true).await + } + + async fn start_plaintext_gateway_listener_with_service_routing( + enabled: bool, + ) -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") .await .expect("failed to bind test listener"); let listen_addr = listener.local_addr().expect("failed to read local addr"); - let state = test_state(listen_addr, false).await; + let mut state = test_state(listen_addr, false).await; + Arc::get_mut(&mut state) + .unwrap() + .config + .service_routing + .enabled = enabled; let service = MultiplexService::new(state); let (shutdown_tx, shutdown_rx) = watch::channel(false); let handle = tokio::spawn(serve_gateway_listener( @@ -3146,6 +3157,22 @@ mod tests { (listen_addr, shutdown_tx, handle) } + #[tokio::test] + async fn gateway_routing_opt_out_keeps_control_plane_available() { + let (addr, shutdown, handle) = + start_plaintext_gateway_listener_with_service_routing(false).await; + let mut client = OpenShellClient::connect(format!("http://{addr}")) + .await + .unwrap(); + client.health(HealthRequest {}).await.unwrap(); + let response = send_plain_http(addr, format!( + "GET /app HTTP/1.1\r\nHost: default--demo--web.openshell.localhost:{}\r\nConnection: close\r\n\r\n", addr.port() + )).await; + assert!(response.starts_with("HTTP/1.1 404")); + assert!(response.contains("Gateway service routing is disabled")); + stop_listener(shutdown, handle).await; + } + #[tokio::test] async fn production_gateway_listener_serves_reflection() { use tonic_reflection::pb::v1::{ diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index 4d8c4a2ace..5c8ff75a43 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -9,6 +9,8 @@ use axum::{ }; use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header}; use hyper_util::rt::TokioIo; +pub use openshell_core::app_listener::service_route_hash_label; +use openshell_core::app_listener::service_route_label; use openshell_core::config::ServiceRoutingConfig; use openshell_core::proto::{ Sandbox, SandboxPhase, ServiceAuthorizationMode, ServiceEndpoint, TcpRelayTarget, relay_open, @@ -19,7 +21,6 @@ use openshell_ocsf::{ HttpActivityBuilder, HttpRequest, HttpResponse as OcsfHttpResponse, NetworkActivityBuilder, OcsfEvent, SeverityId, StateId, StatusId, Url as OcsfUrl, }; -use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; @@ -162,6 +163,9 @@ pub fn endpoint_url( sandbox: &str, service: &str, ) -> Option { + if !config.service_routing.enabled { + return None; + } let host = endpoint_host( config.service_routing_discriminator(), &config.service_routing, @@ -198,79 +202,12 @@ fn endpoint_host( service: &str, ) -> Option { let base_domain = config.base_domains.first()?; - Some(discriminator.map_or_else( - || { - format!( - "{}.{base_domain}", - service_route_readable_label(workspace, sandbox, service) - ) - }, - |gateway_name| { - format!( - "{}.{}", - service_route_hash_label( - gateway_name, - workspace, - sandbox, - (!service.is_empty()).then_some(service), - ), - base_domain - ) - }, + Some(format!( + "{}.{base_domain}", + service_route_label(discriminator, workspace, sandbox, service) )) } -/// Readable DNS label shared by discovery and gateway routing. -pub fn service_route_readable_label(workspace: &str, sandbox: &str, service: &str) -> String { - if service.is_empty() { - format!("{workspace}--{sandbox}") - } else { - format!("{workspace}--{sandbox}--{service}") - } -} - -/// Returns the stable, one-label DNS name for a service route. -pub fn service_route_hash_label( - gateway_name: &str, - workspace: &str, - sandbox: &str, - service: Option<&str>, -) -> String { - let mut hasher = Sha256::new(); - hasher.update(gateway_name.as_bytes()); - hasher.update([0]); - hasher.update(workspace.as_bytes()); - hasher.update([0]); - hasher.update(sandbox.as_bytes()); - hasher.update([0]); - hasher.update(service.unwrap_or("").as_bytes()); - let digest = hasher.finalize(); - base32_lower_no_padding(&digest[..10]) -} - -fn base32_lower_no_padding(bytes: &[u8]) -> String { - const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; - let mut output = String::with_capacity(16); - let mut accumulator = 0_u32; - let mut bits = 0_u8; - for byte in bytes { - accumulator = (accumulator << 8) | u32::from(*byte); - bits += 8; - while bits >= 5 { - bits -= 5; - output.push(char::from( - ALPHABET[((accumulator >> bits) & 0x1f) as usize], - )); - } - } - if bits > 0 { - output.push(char::from( - ALPHABET[((accumulator << (5 - bits)) & 0x1f) as usize], - )); - } - output -} - /// A resolved service-route destination. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ServiceRouteTarget { @@ -576,6 +513,12 @@ pub async fn proxy_sandbox_service_request( state: Arc, req: Request, ) -> impl IntoResponse { + if !state.config.service_routing.enabled { + return service_error_response( + StatusCode::NOT_FOUND, + "Gateway service routing is disabled", + ); + } let Some(host) = request_host(&req).map(str::to_string) else { return StatusCode::NOT_FOUND.into_response(); }; @@ -1432,6 +1375,64 @@ mod tests { use super::*; use openshell_core::config::ServiceHostnameFormat; + #[tokio::test] + async fn disabled_gateway_routing_rejects_http_and_websocket_on_both_routers() { + use http_body_util::BodyExt as _; + use tower::ServiceExt as _; + + for hashed in [false, true] { + for listener_enabled in [false, true] { + let mut state = crate::grpc::test_support::test_server_state().await; + let inner = Arc::get_mut(&mut state).unwrap(); + inner.config.service_routing.enabled = false; + inner.config.supervisor_app_listener.enabled = listener_enabled; + inner.config.name = "gateway-a".into(); + inner.config.service_routing.service_hostname_format = if hashed { + ServiceHostnameFormat::Hashed + } else { + ServiceHostnameFormat::Readable + }; + inner.service_route_index = + Arc::new(ServiceRouteIndex::new(Some("gateway-a".into()))); + inner + .service_route_index + .register("default", "my-sandbox", "web") + .unwrap(); + let host = if hashed { + "enh47uqyj7bpyttz.openshell.localhost" + } else { + "default--my-sandbox--web.openshell.localhost" + }; + for service_only in [false, true] { + for websocket in [false, true] { + let router = if service_only { + crate::service_http_router(state.clone()) + } else { + crate::http_router(state.clone()) + }; + let mut request = Request::builder().uri("/app").header(header::HOST, host); + if websocket { + request = request + .header(header::CONNECTION, "upgrade") + .header(header::UPGRADE, "websocket") + .header("sec-websocket-version", "13") + .header("sec-websocket-key", "dGhlIHNhbXBsZSBub25jZQ=="); + } + let response = router + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response.into_body().collect().await.unwrap().to_bytes(), + "Gateway service routing is disabled" + ); + } + } + } + } + } + fn endpoint() -> ServiceEndpoint { ServiceEndpoint { metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { @@ -1536,6 +1537,47 @@ mod tests { ); } + #[test] + fn switching_ingress_preserves_public_urls_in_both_hostname_modes() { + for format in [ + ServiceHostnameFormat::Readable, + ServiceHostnameFormat::Hashed, + ] { + let mut cfg = openshell_core::Config::new(Some(tls_config())) + .with_bind_address("0.0.0.0:443".parse().unwrap()) + .with_server_sans(["*.apps.example.com"]) + .with_name("gateway-a") + .with_service_hostname_format(format); + cfg.supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + base_url: Some("https://apps.example.com".into()), + }; + for workspace in ["default", "team-a"] { + for service in ["", "web"] { + let application_url = cfg + .supervisor_app_listener + .service_url( + workspace, + "my-sandbox", + service, + cfg.service_routing_discriminator(), + ) + .unwrap(); + // Switching to gateway ingress removes listener configuration, + // but preserves the installation identity and public origin. + let listener = std::mem::take(&mut cfg.supervisor_app_listener); + let gateway_url = endpoint_url(&cfg, workspace, "my-sandbox", service).unwrap(); + assert_eq!( + application_url, gateway_url, + "{format:?}: {workspace}/{service}" + ); + cfg.supervisor_app_listener = listener; + } + } + } + } + #[test] fn service_route_hash_label_uses_stable_fixed_vectors() { assert_eq!( diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 723cbed928..f7149f2c3a 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -179,7 +179,7 @@ mod tests { // updates use a separate capability/service and add one classified RPC. // Discovery RPCs move to ServiceDiscovery; message and durable closures are unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "daa132037ee47c7a46b08dfab9bab379961bc06ff3e9ebbb329809b724f2a4f6"; + "9570c5a65c405284cecb1d14067f91be2ce5d5dc98f2ceb0d7504ded776a863c"; const DURABLE_SCHEMA_SHA256: &str = "cb15f8f79a855064a43fe8c6cd908bf07eca6604e2bf8ab7d2de0ef33e97da4e"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -726,7 +726,7 @@ mod tests { overlap_hash.as_str(), ), ( - (344, 31), + (346, 31), (100, 22), (86, 22), PUBLIC_RPC_SCHEMA_SHA256, diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index da6afbf271..856cc0b907 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -291,8 +291,10 @@ five seconds, even if the configured polling interval is longer. Failed polls do not renew routes. Ending the configuration loop suspends routes. `OPENSHELL_APP_LISTENER_ENABLED=true` explicitly enables the process listener. -`OPENSHELL_APP_LISTEN_ADDRESS`, `OPENSHELL_APP_SERVICE_DOMAIN`, -`OPENSHELL_APP_TLS_CERT`, and `OPENSHELL_APP_TLS_KEY` are then required. Hosted +`OPENSHELL_APP_LISTEN_ADDRESS`, `OPENSHELL_APP_BASE_URL`, +`OPENSHELL_APP_TLS_CERT`, and `OPENSHELL_APP_TLS_KEY` are then required. The domain +is derived from the HTTPS base URL; its public port does not change the listen +address. Hosted instances receive the process router explicitly and never read another instance's configuration from process environment. TLS configuration uses `openshell-crypto`; certificates are read at process startup. Restart the @@ -332,3 +334,9 @@ seconds from issuance, independently of configuration freshness. Expiry closes streams; later renewal cannot revive them. Takeover preserves the workload and public hostname while replacing the app endpoint. Clients reconnect after an interruption; sockets are not migrated. + +The gateway supplies a canonical `application_hostname` with each installed +service route. The listener dispatches exact hashed or readable SNI names to +the correct sandbox instance, validates HTTP Host against SNI, and fences old +streams when a hostname changes. Readable aliases remain accepted during +migration; an older gateway omitting the field retains readable routing. diff --git a/crates/openshell-supervisor/src/app_listener.rs b/crates/openshell-supervisor/src/app_listener.rs index 9b7098aed6..7f5acaf426 100644 --- a/crates/openshell-supervisor/src/app_listener.rs +++ b/crates/openshell-supervisor/src/app_listener.rs @@ -57,24 +57,26 @@ impl Config { } } let address = read("OPENSHELL_APP_LISTEN_ADDRESS"); - let domain = read("OPENSHELL_APP_SERVICE_DOMAIN"); + let base_url = read("OPENSHELL_APP_BASE_URL"); let certificate = read("OPENSHELL_APP_TLS_CERT"); let key = read("OPENSHELL_APP_TLS_KEY"); - let (Some(address), Some(domain), Some(certificate), Some(key)) = - (address, domain, certificate, key) + let (Some(address), Some(base_url), Some(certificate), Some(key)) = + (address, base_url, certificate, key) else { return Err(miette::miette!( - "app listener requires address, domain, certificate, and key" + "app listener requires address, base_url, certificate, and key" )); }; let address: SocketAddr = address.parse().into_diagnostic()?; - openshell_core::app_listener::AppListenerConfig { + let listener = openshell_core::app_listener::AppListenerConfig { enabled: true, port: address.port(), - domain: domain.clone(), - } - .validate() - .map_err(|error| miette::miette!(error))?; + base_url: Some(base_url), + }; + listener + .validate() + .map_err(|error| miette::miette!(error))?; + let domain = listener.domain(); Ok(Some(Self { address, domain, @@ -280,7 +282,7 @@ impl Routes { return; } if let Some(registration) = &self.registration - && !registration.bind(&result.workspace, &self.stopped) + && !registration.bind(&result.workspace, &routes, &self.stopped) { self.suspend(); return; @@ -366,15 +368,19 @@ impl Routes { domain: &str, ) -> Option<(SandboxServiceRoute, RouteToken)> { let encoded = sni.strip_suffix(&format!(".{domain}"))?; - let service = if encoded == prefix { - "" - } else { - encoded.strip_prefix(&format!("{prefix}--"))? - }; let snapshot = self.sender.borrow(); if self.serving_expiry(&snapshot)? <= Instant::now() { return None; } + let service = if let Some(route) = snapshot.routes.values().find(|route| { + !route.application_hostname.is_empty() && route.application_hostname == sni + }) { + route.name.as_str() + } else if encoded == prefix { + "" + } else { + encoded.strip_prefix(&format!("{prefix}--"))? + }; Some(( snapshot.routes.get(service)?.clone(), RouteToken { @@ -432,12 +438,38 @@ struct Registration { router: Router, sandbox: String, prefix: Mutex, + hostnames: Mutex>, sender: mpsc::Sender, } impl Registration { - fn bind(&self, workspace: &str, stopped: &AtomicBool) -> bool { + fn bind( + &self, + workspace: &str, + routes: &BTreeMap, + stopped: &AtomicBool, + ) -> bool { let next = format!("{workspace}--{}", self.sandbox); + let mut names = vec![next.clone()]; + for route in routes.values() { + if route.application_hostname.is_empty() { + continue; // Older gateways only supplied the readable route identity. + } + let Some(label) = route + .application_hostname + .strip_suffix(&format!(".{}", self.router.domain)) + else { + return false; + }; + if !valid_label(label) || route.application_hostname.len() > 253 { + return false; + } + names.push(route.application_hostname.clone()); + } + names.sort(); + if names.windows(2).any(|pair| pair[0] == pair[1]) { + return false; + } let mut prefix = self .prefix .lock() @@ -448,45 +480,62 @@ impl Registration { if stopped.load(Ordering::Acquire) { return false; } + let mut hostnames = self + .hostnames + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let mut entries = self .router .registrations .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if entries - .get(&next) - .is_some_and(|sender| !sender.same_channel(&self.sender)) - { + // Check the whole replacement before changing any registration. An exact + // hashed label must never select an arbitrary sandbox on a shared host. + if names.iter().any(|name| { + entries + .get(name) + .is_some_and(|sender| !sender.same_channel(&self.sender)) + }) { return false; } - if *prefix != next { + for name in hostnames.iter() { if entries - .get(&*prefix) - .is_some_and(|s| s.same_channel(&self.sender)) + .get(name) + .is_some_and(|sender| sender.same_channel(&self.sender)) { - entries.remove(&*prefix); + entries.remove(name); } - prefix.clone_from(&next); } - entries.insert(next, self.sender.clone()); + for name in &names { + entries.insert(name.clone(), self.sender.clone()); + } + *prefix = next; + *hostnames = names; true } fn remove(&self) { - let prefix = self + // Use the same lock as bind's stopped check, including exact aliases. + let _prefix = self .prefix .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + let hostnames = self + .hostnames + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let mut entries = self .router .registrations .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if entries - .get(&*prefix) - .is_some_and(|s| s.same_channel(&self.sender)) - { - entries.remove(&*prefix); + for name in hostnames.iter() { + if entries + .get(name) + .is_some_and(|sender| sender.same_channel(&self.sender)) + { + entries.remove(name); + } } } } @@ -553,6 +602,7 @@ impl Router { router: self.clone(), sandbox: sandbox.to_string(), prefix: Mutex::new(String::new()), + hostnames: Mutex::new(Vec::new()), sender, }); Ok(InstanceListener { @@ -592,10 +642,15 @@ impl BoundListener { let Ok(Ok(tls)) = tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await else { return; }; let Some(sni) = tls.get_ref().1.server_name() else { return; }; let Some(label) = sni.strip_suffix(&format!(".{domain}")) else { return; }; - let mut segments = label.split("--"); - let (Some(workspace), Some(sandbox)) = (segments.next(), segments.next()) else { return; }; - let prefix = format!("{workspace}--{sandbox}"); - let sender = dispatch.registrations.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&prefix).cloned(); + let sender = { + let entries = dispatch.registrations.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + entries.get(sni).cloned().or_else(|| { + // Keep readable aliases during migration, as gateway routing does. + let mut segments = label.split("--"); + let (Some(workspace), Some(sandbox)) = (segments.next(), segments.next()) else { return None; }; + entries.get(&format!("{workspace}--{sandbox}")).cloned() + }) + }; if let Some(sender) = sender { let _ = sender.try_send(Accepted { tls, permit }); } }); } @@ -958,6 +1013,7 @@ mod tests { target_port: 8080, authorization_mode: 1, resource_version: 1, + application_hostname: String::new(), }; let mut config = installed(vec![route.clone()]); config.config_revision = 42; @@ -996,6 +1052,7 @@ mod tests { target_port: 8080, authorization_mode: 1, resource_version: 1, + application_hostname: String::new(), }; routes.install(&installed(vec![route.clone()])); assert!( @@ -1039,7 +1096,10 @@ mod tests { fn listener_requires_explicit_enablement_and_complete_configuration() { let complete = BTreeMap::from([ ("OPENSHELL_APP_LISTEN_ADDRESS", "0.0.0.0:9443"), - ("OPENSHELL_APP_SERVICE_DOMAIN", "services.example.com"), + ( + "OPENSHELL_APP_BASE_URL", + "https://SERVICES.example.com:10443", + ), ("OPENSHELL_APP_TLS_CERT", "/operator/cert.pem"), ("OPENSHELL_APP_TLS_KEY", "/operator/key.pem"), ]); @@ -1056,14 +1116,11 @@ mod tests { .is_none() ); values.insert("OPENSHELL_APP_LISTENER_ENABLED", "true"); - assert_eq!( - Config::from_values(|name| values.get(name).map(ToString::to_string)) - .unwrap() - .unwrap() - .address - .port(), - 9443 - ); + let configured = Config::from_values(|name| values.get(name).map(ToString::to_string)) + .unwrap() + .unwrap(); + assert_eq!(configured.address.port(), 9443); + assert_eq!(configured.domain, "services.example.com"); values.remove("OPENSHELL_APP_TLS_KEY"); assert!(Config::from_values(|name| values.get(name).map(ToString::to_string)).is_err()); assert!( @@ -1232,6 +1289,7 @@ mod tests { target_port: 8080, authorization_mode: 1, resource_version: 1, + application_hostname: String::new(), }; routes.install(&installed(vec![route.clone()])); let connector = Arc::new(Connector { @@ -1357,6 +1415,96 @@ mod tests { ); } + #[tokio::test] + async fn hashed_sni_routes_to_the_correct_instance_and_replacement_withdraws_old_names() { + let f = fixture(); + let hostname = |gateway, sandbox, service| { + format!( + "{}.svc.openshell.internal", + openshell_core::app_listener::service_route_hash_label( + gateway, + "default", + sandbox, + Some(service) + ) + ) + }; + let first_host = hostname("gateway-a", "sandbox-a", "web"); + let mut first = f.route.clone(); + first.application_hostname.clone_from(&first_host); + f.routes.install(&installed(vec![first.clone()])); + assert!( + request(&f, &first_host, &first_host, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert!( + request(&f, &first_host, HOST, "") + .await + .starts_with("HTTP/1.1 421") + ); + // Readable migration aliases remain usable, as with gateway-mediated hosts. + assert!( + request(&f, HOST, HOST, "") + .await + .starts_with("HTTP/1.1 200") + ); + let before = f.connector.calls.load(Ordering::SeqCst); + let foreign = hostname("gateway-b", "sandbox-a", "web"); + assert!( + !request(&f, &foreign, &foreign, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), before); + + let second = f.host.router().instance("sandbox-b").unwrap(); + let second_routes = second.routes.clone(); + let connector = Arc::new(Connector { + calls: AtomicUsize::new(0), + }); + let handle = second.start(rpc_test::loopback(connector.clone())); + // A canonical hostname collision cannot steal another instance's mapping. + second_routes.install(&installed(vec![first.clone()])); + assert!(second_routes.sender.borrow().routes.is_empty()); + let second_host = hostname("gateway-a", "sandbox-b", "web"); + second_routes.install(&installed(vec![SandboxServiceRoute { + application_hostname: second_host.clone(), + endpoint_id: "endpoint-b".into(), + ..first.clone() + }])); + assert!( + request(&f, &second_host, &second_host, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert_eq!(connector.calls.load(Ordering::SeqCst), 1); + let (_, token) = f + .routes + .resolve(&first_host, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + let mut replacement = first.clone(); + replacement.application_hostname.clone_from(&foreign); + f.routes.install(&installed(vec![replacement])); + assert!(!f.routes.valid(&first, token)); + assert!( + !request(&f, &first_host, &first_host, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert!( + request(&f, &foreign, &foreign, "") + .await + .starts_with("HTTP/1.1 200") + ); + drop(handle); + assert!( + !request(&f, &second_host, &second_host, "") + .await + .starts_with("HTTP/1.1 200") + ); + } + #[tokio::test] async fn shared_listener_keeps_instance_connectors_and_lifetimes_separate() { let f = fixture(); @@ -1439,7 +1587,18 @@ mod tests { }); let mut installing_routes = f.routes.clone(); installing_routes.install_probe = Some(probe.clone()); - let snapshot = installed(vec![f.route.clone()]); + let hostname = format!( + "{}.svc.openshell.internal", + openshell_core::app_listener::service_route_hash_label( + "gateway-a", + "default", + "sandbox-a", + Some("web") + ) + ); + let mut route = f.route.clone(); + route.application_hostname.clone_from(&hostname); + let snapshot = installed(vec![route.clone()]); let installer = std::thread::spawn(move || installing_routes.install(&snapshot)); probe.checked_running.wait(); // Complete shutdown after install's optimistic stopped check but @@ -1455,12 +1614,20 @@ mod tests { .unwrap() .contains_key("default--sandbox-a") ); + assert!( + !f.host + .router() + .registrations + .lock() + .unwrap() + .contains_key(&hostname) + ); let replacement = f.host.router().instance("sandbox-a").unwrap(); - replacement.routes.install(&installed(vec![f.route])); + replacement.routes.install(&installed(vec![route])); assert!( replacement .routes - .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .resolve(&hostname, "default--sandbox-a", "svc.openshell.internal") .is_some(), "the stopped instance must not prevent its replacement from serving" ); @@ -1475,6 +1642,7 @@ mod tests { target_port: 8080, authorization_mode: 1, resource_version: 1, + application_hostname: String::new(), }; routes.install(&installed(vec![route.clone()])); for _ in 0..10 { @@ -1587,47 +1755,68 @@ mod tests { assert!(!headers.contains_key("x-internal")); } #[tokio::test] - async fn websocket_echo_closes_on_route_withdrawal() { - let f = fixture(); - let stream = openshell_core::net::connect_tcp_nodelay_best_effort(f.address) - .await - .unwrap(); - let name = rustls::pki_types::ServerName::try_from(HOST.to_string()).unwrap(); - let tls = tokio_rustls::TlsConnector::from(f.tls.clone()) - .connect(name, stream) - .await - .unwrap(); - let mut request = format!("wss://{HOST}/echo").into_client_request().unwrap(); - request - .headers_mut() - .insert(header::ORIGIN, format!("https://{HOST}").parse().unwrap()); - let (mut ws, response) = tokio_tungstenite::client_async(request, tls).await.unwrap(); - assert_eq!(response.headers()["x-test-upstream-host"], HOST); - assert_eq!( - response.headers()["x-test-upstream-origin"], - format!("https://{HOST}") - ); - ws.send(tokio_tungstenite::tungstenite::Message::Text( - "hello".into(), - )) - .await - .unwrap(); - assert_eq!( - tokio::time::timeout(Duration::from_secs(2), ws.next()) + async fn readable_and_hashed_websocket_echo_closes_on_route_withdrawal() { + for hashed in [false, true] { + let f = fixture(); + let hostname = if hashed { + format!( + "{}.svc.openshell.internal", + openshell_core::app_listener::service_route_hash_label( + "gateway-a", + "default", + "sandbox-a", + Some("web") + ) + ) + } else { + HOST.to_string() + }; + let mut route = f.route.clone(); + route.application_hostname.clone_from(&hostname); + f.routes.install(&installed(vec![route])); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(f.address) .await - .unwrap() - .unwrap() - .unwrap() - .to_text() - .unwrap(), - "hello" - ); - f.routes.suspend(); - let closed = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .unwrap(); + let name = rustls::pki_types::ServerName::try_from(hostname.clone()).unwrap(); + let tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(name, stream) + .await + .unwrap(); + let mut request = format!("wss://{hostname}/echo") + .into_client_request() + .unwrap(); + request.headers_mut().insert( + header::ORIGIN, + format!("https://{hostname}").parse().unwrap(), + ); + let (mut ws, response) = tokio_tungstenite::client_async(request, tls).await.unwrap(); + assert_eq!(response.headers()["x-test-upstream-host"], hostname); + assert_eq!( + response.headers()["x-test-upstream-origin"], + format!("https://{hostname}") + ); + ws.send(tokio_tungstenite::tungstenite::Message::Text( + "hello".into(), + )) .await .unwrap(); - assert!(closed.is_none() || closed.unwrap().is_err()); - assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap() + .unwrap() + .unwrap() + .to_text() + .unwrap(), + "hello" + ); + f.routes.suspend(); + let closed = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap(); + assert!(closed.is_none() || closed.unwrap().is_err()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } } type TestWebSocket = diff --git a/deploy/helm/openshell/ci/values-app-listener.yaml b/deploy/helm/openshell/ci/values-app-listener.yaml index 2604e1a6ff..3f55850fbd 100644 --- a/deploy/helm/openshell/ci/values-app-listener.yaml +++ b/deploy/helm/openshell/ci/values-app-listener.yaml @@ -9,6 +9,6 @@ gatewayConfig: openshell.supervisor.app_listener: enabled: true port: 9443 - domain: services.example.com + base_url: https://services.example.com openshell.drivers.kubernetes.sandbox_runtime.app_listener: tls_secret_name: openshell-app-tls diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index 513f49d283..c24650c64f 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -13,7 +13,7 @@ {{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml }} {{- $appPort := get $app "port" | default 8443 }} {{- if get $app "enabled" }} -{{- $_ := required "supervisor app listener requires domain" (get $app "domain") }} +{{- $_ := required "supervisor app listener requires base_url" (get $app "base_url") }} {{- $_ := required "supervisor app listener requires tls_secret_name" (get $appDeployment "tls_secret_name") }} {{- if or (lt (int $appPort) 1024) (gt (int $appPort) 65535) (eq (int $appPort) 5501) (eq (int $appPort) (int (get $shared "runtime_port"))) }} {{- fail "supervisor app listener port must be in 1024..65535 and distinct from runtime and health ports" }} @@ -101,8 +101,8 @@ spec: value: "true" - name: OPENSHELL_APP_LISTEN_ADDRESS value: {{ printf "0.0.0.0:%v" $appPort | quote }} - - name: OPENSHELL_APP_SERVICE_DOMAIN - value: {{ get $app "domain" | quote }} + - name: OPENSHELL_APP_BASE_URL + value: {{ get $app "base_url" | quote }} - name: OPENSHELL_APP_TLS_CERT value: /etc/openshell/app-tls/tls.crt - name: OPENSHELL_APP_TLS_KEY diff --git a/deploy/helm/openshell/tests/app_listener_test.yaml b/deploy/helm/openshell/tests/app_listener_test.yaml index df59503b76..7aefe2d1c7 100644 --- a/deploy/helm/openshell/tests/app_listener_test.yaml +++ b/deploy/helm/openshell/tests/app_listener_test.yaml @@ -15,7 +15,7 @@ tests: openshell.supervisor.app_listener: enabled: false port: 9443 - domain: services.example.com + base_url: https://services.example.com openshell.drivers.kubernetes.sandbox_runtime.app_listener: tls_secret_name: app-tls asserts: @@ -39,7 +39,7 @@ tests: openshell.supervisor.app_listener: enabled: true port: 9443 - domain: services.example.com + base_url: https://services.example.com:10443 openshell.drivers.kubernetes.sandbox_runtime.app_listener: tls_secret_name: app-tls asserts: @@ -51,6 +51,12 @@ tests: name: OPENSHELL_APP_LISTEN_ADDRESS value: 0.0.0.0:9443 documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_APP_BASE_URL + value: https://services.example.com:10443 + documentIndex: 0 - contains: path: spec.template.spec.volumes content: @@ -66,7 +72,7 @@ tests: openshell.supervisor.app_listener: enabled: true port: 5500 - domain: services.example.com + base_url: https://services.example.com openshell.drivers.kubernetes.sandbox_runtime.app_listener: tls_secret_name: app-tls asserts: diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 157ece486b..26d4e3f438 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -13,6 +13,24 @@ release: namespace: my-namespace tests: + - it: leaves gateway service routing enabled by the runtime default when omitted + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'service_routing_enabled\s*=' + + - it: disables gateway service routing through canonical configuration + template: templates/gateway-config.yaml + set: + gatewayConfig: + openshell.gateway: + service_routing_enabled: false + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^service_routing_enabled\s*=\s*false$' + - it: disables the WebSocket tunnel unless explicitly enabled for an edge proxy template: templates/gateway-config.yaml asserts: diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 45f4f457fd..3f75202c82 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -353,6 +353,8 @@ config_delivery_mode = "poll" # Wildcard DNS SANs (e.g. "*.dev.openshell.localhost") also enable sandbox # service URLs under that domain. server_sans = ["openshell", "*.dev.openshell.localhost"] +# Publish gateway service URLs and forward application traffic through the gateway. +service_routing_enabled = true # Allow plaintext HTTP routing for loopback sandbox service URLs. enable_loopback_service_http = true # Hostname format for exposed sandbox services. "hashed" makes service URLs @@ -605,6 +607,25 @@ and `https://--./` for the unnamed endpoint. The base domain is the first wildcard DNS SAN in `server_sans`; gateways without SANs use the built-in `openshell.localhost`. +`service_routing_enabled` defaults to `true`. Set it to `false` to stop publishing +gateway service URLs and reject gateway-mediated application HTTP requests and +WebSocket upgrades with `404`. Control-plane APIs, service declarations, direct +supervisor listeners, and discovery remain available. The setting also accepts +`--service-routing-enabled=false` or `OPENSHELL_SERVICE_ROUTING_ENABLED=false`, +with CLI > environment > TOML > default precedence. In Helm, configure the +canonical gateway table: + +```yaml +gatewayConfig: + openshell.gateway: + service_routing_enabled: false +``` + +With gateway routing disabled, `gateway_url` is empty and `gateway_service_urls` +is empty. A configured application origin supplies the preferred URL; without +one, preferred URLs are empty. Idempotent retries preserve recorded preferred +URLs; explicit application/gateway fields reflect the current configuration. + `service_hostname_format` selects the hostname format for exposed sandbox services when multiple gateways share one service base domain behind a single ingress and wildcard certificate. Identically named workspaces, sandboxes, and @@ -2039,14 +2060,19 @@ ingress remains available independently. [openshell.supervisor.app_listener] enabled = true port = 9443 -domain = "services.example.com" +base_url = "https://services.example.com" [openshell.drivers.kubernetes.sandbox_runtime.app_listener] tls_secret_name = "openshell-app-tls" ``` -The listener defaults to disabled, with port `8443` and no domain. Port, domain, -or driver options alone do not enable it. Use an unprivileged port distinct from +`base_url` replaces the separate `domain` and `public_url` settings. Remove +those fields when updating an existing app-listener configuration; the domain is +now derived from `base_url`. + +The listener defaults to disabled, with port `8443` and no base URL. Port, +`base_url`, or driver options alone do not enable it. An enabled listener requires +`base_url`; its hostname supplies the service domain. Use an unprivileged port distinct from the supervisor runtime and health ports. The common `port` controls both the listener and its Kubernetes endpoint. @@ -2070,9 +2096,25 @@ Expose services with the existing commands. For direct listener testing, an unnamed service on sandbox `demo` in workspace `default` uses SNI and Host `default--demo.services.example.com`; a service named `web` uses `default--demo--web.services.example.com`. Configure DNS or your ingress to reach -the owning supervisor while preserving that hostname. The discovery API supplies accepted supervisor endpoints. Application URL -publication is separate work; current CLI URLs still identify the existing -gateway path. +the owning supervisor while preserving that hostname. + +Both ingress paths use `[openshell.gateway].service_hostname_format`; the app +listener has no separate naming mode. Keeping the same installation name and +public HTTPS domain/port preserves the URL when switching between gateway and +app-listener ingress. + +With `[openshell.gateway].service_hostname_format = "hashed"`, application URLs +and discovery use the same 16-character label as gateway-mediated URLs. The +label includes the gateway installation name, workspace, sandbox, and service; +application and gateway origins may still have different domains and ports. +Use a unique gateway `name` among installations sharing an application domain. +Supervisors install the gateway-declared hostname and retain readable aliases +for migration. Proxies and controllers consume discovery hostnames as returned. +Upgrade gateways and supervisors together before enabling hashed application +hostnames; older supervisors cannot serve the new hashed routes. + +The discovery API supplies accepted supervisor endpoints. Configure `base_url` +to publish the application origin; otherwise CLI URLs retain the gateway path. Enabled supervisors report installed service routes through their authenticated gateway sessions. The gateway accepts only current configuration and ownership, @@ -2125,3 +2167,26 @@ does not create a new lease when serving discovery. Watches recheck bearer credentials and workspace access every second and terminate if either is lost. OIDC token revocation follows the configured verifier's semantics; offline JWT verification cannot detect issuer-side revocation before token expiry. + +Set `base_url` to the external HTTPS base origin under which service records +are created. The service hostname is `.`. The +base URL must have a DNS hostname and cannot contain credentials, a path, query, +or fragment. A nondefault public port is supported, for example +`https://services.example.com:10443`. This port is independent of the internal +listener `port` and the gateway API address. Set `enabled = true` to activate +the listener; configuring `base_url` alone does not enable routing or create DNS, +certificates, or ingress. + +When configured, service responses return the application address as `url` and +`application_url`, with `gateway_url` available as a separate alternative. +Create-time exposure returns corresponding preferred, application, and gateway +URL maps. CLI and SDK callers preserve the application origin, including its +port, when the gateway is reached through a port-forward. With the listener +disabled, the existing gateway URL behavior remains. Idempotent retries preserve the +original `url` and `service_urls` values. The explicit application and gateway +fields use the current URL configuration. A configured URL does not mean that a +supervisor installed the route or ingress is ready. + +Upgrade the CLI alongside the gateway when enabling `base_url`. Older CLI +versions can replace the public application port with the gateway connection's +port. diff --git a/e2e/kubernetes/app-listener/run.py b/e2e/kubernetes/app-listener/run.py index 523f8144ac..a5f6bf7583 100644 --- a/e2e/kubernetes/app-listener/run.py +++ b/e2e/kubernetes/app-listener/run.py @@ -196,7 +196,7 @@ def wait_for_shared_hosts(): "openshell.supervisor.app_listener": { "enabled": True, "port": 9443, - "domain": "services.example.com", + "base_url": "https://services.example.com", }, "openshell.drivers.kubernetes.sandbox_runtime.app_listener": { "tls_secret_name": "openshell-app-tls", diff --git a/proto/openshell.proto b/proto/openshell.proto index 198990a364..74b08e15a5 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1609,7 +1609,14 @@ message SandboxResponse { Sandbox sandbox = 1; // Service URLs created by CreateSandbox, keyed by service name. The empty // key identifies the unnamed service. Other sandbox RPCs return an empty map. + // Idempotent retries preserve the recorded URLs; the explicit access-path + // maps below use the current configuration. map service_urls = 2; + // Configured application origins for the same services. These are + // authoritative public addresses, not installation or health evidence. + map application_service_urls = 3; + // Gateway-routed alternatives, independent of application ingress. + map gateway_service_urls = 4; } // List sandboxes response. @@ -1992,10 +1999,17 @@ message ServiceEndpoint { ServiceAuthorizationMode authorization_mode = 7; } -// Response containing a service endpoint and, when available, its local URL. +// Service addresses. A configured address does not establish readiness. message ServiceEndpointResponse { ServiceEndpoint endpoint = 1; + // Preferred address: application_url when configured, otherwise gateway_url. + // Idempotent retries preserve the originally returned address; the explicit + // access-path fields below use the current configuration. string url = 2; + // Authoritative configured public application address. Preserve its port. + string application_url = 3; + // Gateway-routed alternative; existing gateway routing remains available. + string gateway_url = 4; } // Revoke SSH session request. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index b57c5d4cda..b4d9931dd5 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -490,6 +490,10 @@ message SandboxServiceRoute { uint32 target_port = 3; int32 authorization_mode = 4; uint64 resource_version = 5; + // Canonical application SNI/Host selected by the gateway's hostname format. + // Includes the application domain, without a scheme or port. An empty value + // from an older gateway retains readable workspace--sandbox[--service] routing. + string application_hostname = 6; } // Connection details for one operator-registered supervisor middleware service. diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index 756ff6e44d..e6350a59c0 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -20,6 +20,7 @@ import { POLICY_SOURCE_NAMES, Pushable, SandboxClient, + type SandboxRef, SandboxTemplateClient, SCOPE_NAMES, ServiceAuthorizationMode, @@ -700,6 +701,20 @@ describe('watchLogs', () => { }); describe('create', () => { + it('accepts the existing SandboxRef shape without explicit access-path maps', () => { + const ref: SandboxRef = { + id: 'sb-id', + name: 'sb', + workspace: 'default', + phase: 'ready', + labels: {}, + resourceVersion: '1', + serviceUrls: {}, + restartCount: 0, + }; + expect(ref.serviceUrls).toEqual({}); + }); + it('sends create-time service exposures', async () => { let created: { serviceExposures?: Array<{ service?: string; targetPort?: number; authorizationMode?: number }>; @@ -709,6 +724,8 @@ describe('create', () => { created = req; return { ...readySandbox('sb', 'sb-id'), + applicationServiceUrls: { '': 'https://sb.example.test/', metrics: 'https://metrics.sb.example.test/' }, + gatewayServiceUrls: { '': 'https://sb.gateway.test:8080/' }, serviceUrls: { '': 'https://sb.example.test/', metrics: 'https://metrics.sb.example.test/', @@ -743,6 +760,8 @@ describe('create', () => { authorizationMode: ProtoServiceAuthorizationMode.BEARER_PASSTHROUGH, }, ]); + expect(result.applicationServiceUrls).toEqual(result.serviceUrls); + expect(result.gatewayServiceUrls?.['']).toBe('https://sb.gateway.test:8080/'); expect(result.serviceUrls).toEqual({ '': 'https://sb.example.test/', metrics: 'https://metrics.sb.example.test/', diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index 18eeb2f698..89fa002be6 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -247,6 +247,8 @@ export interface SandboxRef { createdFromWorkloadTemplate?: SandboxWorkloadTemplateProvenance; /** Service URLs returned by creation, keyed by service name. */ serviceUrls: Record; + applicationServiceUrls?: Record; + gatewayServiceUrls?: Record; restartCount: number; /** Creation time in epoch milliseconds; absent when the gateway omits it. */ createdAtMs?: number; @@ -625,7 +627,12 @@ function policySourceName(s: PolicySource): PolicySourceName { return POLICY_SOURCE_NAMES[s] ?? 'unspecified'; } -function sandboxRef(sandbox: Sandbox | undefined, serviceUrls: Record = {}): SandboxRef { +function sandboxRef( + sandbox: Sandbox | undefined, + serviceUrls: Record = {}, + applicationServiceUrls: Record = {}, + gatewayServiceUrls: Record = {}, +): SandboxRef { if (!sandbox) throw new SdkError('invalid_config', 'sandbox missing from gateway response'); const meta = sandbox.metadata; if (!meta?.id || !meta.name) { @@ -648,6 +655,8 @@ function sandboxRef(sandbox: Sandbox | undefined, serviceUrls: Record