From 3c8b960d1e98574888acbfd7b7030a3f06eb769d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 20:24:27 -0700 Subject: [PATCH 1/4] feat(guest): support multiple gateway clusters --- docs/deployment.md | 24 ++ dstack/dstack-types/src/lib.rs | 17 + dstack/dstack-util/src/gateway_checker.rs | 37 +- dstack/dstack-util/src/system_setup.rs | 399 +++++++++++++------- dstack/guest-agent/src/guest_api_service.rs | 4 +- dstack/vmm/src/app.rs | 2 + dstack/vmm/src/config.rs | 29 ++ 7 files changed, 357 insertions(+), 155 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index c50ac42e6..964dab30b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -321,6 +321,30 @@ kms_urls = ["https://kms.example.com:9201"] gateway_urls = ["https://gateway.example.com:9202"] ``` +`gateway_urls` is a failover list for one gateway cluster. To register a CVM +with independently operated clusters, configure explicit groups instead. Each +cluster gets a separate WireGuard interface and key pair; their WireGuard +address ranges must not overlap. All clusters must run the gateway app identity +authorized by the CVM's KMS-issued app keys. + +```toml +[cvm] +kms_urls = ["https://kms.example.com:9201"] + +[[cvm.gateway_clusters]] +name = "primary" +urls = [ + "https://gateway-a.example.com:9202", + "https://gateway-b.example.com:9202", +] +required = true + +[[cvm.gateway_clusters]] +name = "secondary" +urls = ["https://gateway-c.example.com:9202"] +required = false +``` + Restart dstack-vmm to apply changes. --- diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index d43baf895..1fce0d808 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -1093,6 +1093,11 @@ pub struct SysConfig { pub kms_urls: Vec, #[serde(default, alias = "tproxy_urls")] pub gateway_urls: Vec, + /// Independently operated gateway clusters. URLs within one entry are + /// failover endpoints for the same cluster. When empty, `gateway_urls` is + /// treated as one legacy cluster. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub gateway_clusters: Vec, /// Backward-compatible input for sys-config files produced by older hosts. #[serde(default, rename = "pccs_url", skip_serializing)] legacy_pccs_url: Option, @@ -1117,6 +1122,18 @@ pub struct SysConfig { pub vm_config: String, } +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct GatewayClusterConfig { + /// Stable local name used for the per-cluster key cache. + pub name: String, + /// Failover RPC endpoints belonging to this cluster. + pub urls: Vec, + /// Whether failure to register this cluster makes the refresh fail. + #[serde(default = "default_true")] + pub required: bool, +} + #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CollateralUrls { diff --git a/dstack/dstack-util/src/gateway_checker.rs b/dstack/dstack-util/src/gateway_checker.rs index e373c46d6..6bb20276f 100644 --- a/dstack/dstack-util/src/gateway_checker.rs +++ b/dstack/dstack-util/src/gateway_checker.rs @@ -28,7 +28,7 @@ //! unit tested without a gateway, a KMS, or a WireGuard interface; all I/O //! lives in [`cmd_gateway_checker`]. -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; @@ -36,7 +36,7 @@ use cmd_lib::run_fun as cmd; use sd_notify::NotifyState; use tracing::{error, info, warn}; -use crate::system_setup::{GatewayRefresher, WG_CONFIG_PATH, WG_INTERFACE}; +use crate::system_setup::GatewayRefresher; /// How often the loop samples the world. const POLL_INTERVAL: Duration = Duration::from_secs(10); @@ -104,7 +104,7 @@ impl Backoff { struct Observation { /// Seconds since the UNIX epoch. now: i64, - /// Whether `/etc/wireguard/dstack-wg0.conf` exists. + /// Whether at least one `/etc/wireguard/dstack-wg*.conf` exists. config_present: bool, /// Most recent handshake as a UNIX timestamp; `None` if the interface has /// never completed one (or does not exist yet). @@ -258,12 +258,28 @@ fn parse_latest_handshake(output: &str) -> Option { .max() } -fn wg_config_present() -> bool { - Path::new(WG_CONFIG_PATH).exists() +fn configured_gateway_interfaces() -> Vec { + let Ok(entries) = std::fs::read_dir("/etc/wireguard") else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter_map(|name| { + name.strip_suffix(".conf") + .filter(|name| name.starts_with("dstack-wg")) + .map(str::to_string) + }) + .collect() } fn observe(now: i64) -> Observation { - let config_present = wg_config_present(); + let interfaces = configured_gateway_interfaces(); + let config_present = !interfaces.is_empty(); + let handshakes = interfaces + .iter() + .map(|interface| latest_handshake(interface)) + .collect::>>(); Observation { now, config_present, @@ -272,9 +288,10 @@ fn observe(now: i64) -> Observation { // the probe matters because that is precisely the state a gateway // outage parks the CVM in: otherwise we would fork `wg` every poll for // the entire outage to answer a question nobody asks. - latest_handshake: config_present - .then(|| latest_handshake(WG_INTERFACE)) - .flatten(), + // The least healthy cluster drives recovery. A missing handshake on + // any configured interface is represented as None; otherwise the + // oldest cluster handshake is the staleness boundary. + latest_handshake: handshakes.and_then(|values| values.into_iter().min()), } } @@ -349,7 +366,7 @@ pub async fn cmd_gateway_checker(args: GatewayCheckerArgs) -> Result<()> { // has been told, which is state this loop should not have to carry. The // consequence is that a boot error stays on the VMM after the checker // recovers, until the VM restarts. - let mut checker = Checker::starting(now_secs(), wg_config_present()); + let mut checker = Checker::starting(now_secs(), !configured_gateway_interfaces().is_empty()); loop { // Ping before the work, not after, so a refresh that never returns // stops the pings. Nothing else can do this for us: a refresh spends diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index b5b9fb7b3..ea4b6d5fc 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -59,7 +59,7 @@ use cert_client::CertRequestClient; use cmd_lib::run_fun as cmd; use dstack_gateway_rpc::{ gateway_client::GatewayClient, PortAttrs as RpcPortAttrs, PortPolicy as RpcPortPolicy, - RegisterCvmRequest, RegisterCvmResponse, WireGuardConfig, WireGuardPeer, + RegisterCvmRequest, RegisterCvmResponse, WireGuardPeer, }; use ra_tls::rcgen::{KeyPair, PKCS_ECDSA_P256_SHA256}; use serde_human_bytes as hex_bytes; @@ -314,9 +314,7 @@ impl HostShared { } const GATEWAY_CACHE_PATH: &str = "/run/dstack/gateway-cache.json"; -/// Name of the WireGuard interface linking this CVM to dstack-gateway. -pub const WG_INTERFACE: &str = "dstack-wg0"; -pub const WG_CONFIG_PATH: &str = "/etc/wireguard/dstack-wg0.conf"; +const GATEWAY_CACHE_PREFIX: &str = "/run/dstack/gateway-cache-"; /// Certificate validity period in seconds (10 days) const CERT_VALIDITY_SECS: u64 = 10 * 24 * 3600; const MAX_SUPPORTED_MANIFEST_VERSION: u32 = 3; @@ -344,7 +342,7 @@ impl GatewayKeyStore { serde_json::from_str(&content).ok() } - fn load() -> Option { + fn load_from_default() -> Option { Self::load_from(Path::new(GATEWAY_CACHE_PATH)) } @@ -354,7 +352,7 @@ impl GatewayKeyStore { Ok(()) } - fn save(&self) -> Result<()> { + fn save_to_default(&self) -> Result<()> { self.save_to(Path::new(GATEWAY_CACHE_PATH)) } @@ -381,6 +379,41 @@ fn gateway_rpc_url(base: &str) -> String { } } +#[derive(Debug)] +struct GatewayTarget { + name: String, + urls: Vec, + required: bool, +} + +struct PreparedGatewayCluster { + name: String, + index: usize, + key_store: GatewayKeyStore, + response: RegisterCvmResponse, +} + +fn validate_disjoint_wireguard_addresses(clusters: &[PreparedGatewayCluster]) -> Result<()> { + let mut addresses = std::collections::HashMap::::new(); + for cluster in clusters { + let wg = cluster.response.wg.as_ref().context("Missing wg info")?; + for address in std::iter::once(wg.client_ip.as_str()) + .chain(wg.servers.iter().map(|peer| peer.ip.as_str())) + { + let address = address.split('/').next().unwrap_or_default().to_string(); + if let Some(other) = addresses.insert(address.clone(), &cluster.name) { + bail!( + "Gateway clusters {} and {} use overlapping WireGuard address {}", + other, + cluster.name, + address + ); + } + } + } + Ok(()) +} + struct GatewayContext<'a> { shared: &'a HostShared, keys: &'a AppKeys, @@ -397,11 +430,12 @@ impl<'a> GatewayContext<'a> { gateway_url: &str, client_key: &str, client_cert: &str, + gateway_app_id: &str, ) -> Result> { let url = gateway_rpc_url(gateway_url); let ca_cert = self.keys.ca_cert.clone(); let cert_validator = AppIdValidator { - allowed_app_id: self.keys.gateway_app_id.clone(), + allowed_app_id: gateway_app_id.to_string(), }; let client = RaClientConfig::builder() .remote_uri(url) @@ -409,7 +443,7 @@ impl<'a> GatewayContext<'a> { .tls_client_key(client_key.to_string()) .tls_ca_cert(ca_cert) .tls_built_in_root_certs(false) - .tls_no_check(self.keys.gateway_app_id == "any") + .tls_no_check(gateway_app_id == "any") .verify_server_attestation(false) .cert_validator(Box::new(move |cert| cert_validator.validate(cert))) .build() @@ -422,6 +456,7 @@ impl<'a> GatewayContext<'a> { &self, gateway_url: &str, key_store: &GatewayKeyStore, + gateway_app_id: &str, ) -> Result { let port_policy = RpcPortPolicy { ports: self @@ -437,8 +472,12 @@ impl<'a> GatewayContext<'a> { .collect(), restrict_mode: self.shared.app_compose.port_policy.restrict_mode, }; - let client = - self.create_gateway_client(gateway_url, &key_store.client_key, &key_store.client_cert)?; + let client = self.create_gateway_client( + gateway_url, + &key_store.client_key, + &key_store.client_cert, + gateway_app_id, + )?; let result = client .register_cvm(RegisterCvmRequest { client_public_key: key_store.wg_pk.clone(), @@ -459,6 +498,7 @@ impl<'a> GatewayContext<'a> { gateway_url, &key_store.client_key, &key_store.client_cert_with_quote, + gateway_app_id, )?; client .register_cvm(RegisterCvmRequest { @@ -471,7 +511,7 @@ impl<'a> GatewayContext<'a> { async fn get_or_generate_key_store(&self) -> Result { // Try to load existing cache - let cache = GatewayKeyStore::load(); + let cache = GatewayKeyStore::load_from_default(); // If cache is fully valid, return it if let Some(ref cache) = cache { @@ -557,6 +597,33 @@ impl<'a> GatewayContext<'a> { }) } + fn key_store_for_additional_cluster( + &self, + name: &str, + certificate_source: &GatewayKeyStore, + ) -> Result<(GatewayKeyStore, PathBuf)> { + let path = PathBuf::from(format!("{GATEWAY_CACHE_PREFIX}{name}.json")); + if let Some(cache) = GatewayKeyStore::load_from(&path) { + if cache.is_cert_valid() { + info!(cluster = name, "Using cached gateway cluster key store"); + return Ok((cache, path)); + } + } + let old = GatewayKeyStore::load_from(&path); + let (wg_sk, wg_pk) = if let Some(old) = old { + (old.wg_sk, old.wg_pk) + } else { + let sk = cmd!(wg genkey)?; + let pk = + cmd!(echo $sk | wg pubkey).or(Err(anyhow!("Failed to generate public key")))?; + (sk, pk) + }; + let mut key_store = certificate_source.clone(); + key_store.wg_sk = wg_sk; + key_store.wg_pk = wg_pk; + Ok((key_store, path)) + } + async fn setup(&self, force: bool) -> Result<()> { if !self.shared.app_compose.gateway_enabled() { info!("dstack-gateway is not enabled"); @@ -566,163 +633,163 @@ impl<'a> GatewayContext<'a> { bail!("Missing allowed dstack-gateway app id"); } - info!("Setting up dstack-gateway"); - - // Get or generate key store (includes WireGuard keys and client certificate) - let key_store = self.get_or_generate_key_store().await?; + let targets = self.gateway_targets()?; + let uses_explicit_clusters = !self.shared.sys_config.gateway_clusters.is_empty(); + info!(clusters = targets.len(), "Setting up dstack-gateway"); - // Persist the key store before attempting registration. Minting it costs a - // KMS round-trip, two cert signing requests and a TDX quote, so a gateway - // outage would otherwise make every retry pay that price again and turn a - // gateway outage into a KMS load spike across the whole fleet. - if let Err(e) = key_store.save() { - warn!("failed to save gateway cache: {e:?}"); + // Certificates are valid for every gateway identity authorized by KMS, + // while each cluster receives a distinct WireGuard identity. + let primary_key_store = self.get_or_generate_key_store().await?; + if let Err(err) = primary_key_store.save_to_default() { + warn!("failed to save gateway cache: {err:?}"); } - if self.shared.sys_config.gateway_urls.is_empty() { - bail!("Missing gateway urls"); - } - // Read config and make API call - let response = 'out: { - let mut error = anyhow!("unknown error"); - for (i, url) in self.shared.sys_config.gateway_urls.iter().enumerate() { - let response = self.register_cvm(url, &key_store).await; - match response { - Ok(response) => { - break 'out response; + let mut prepared = Vec::new(); + let mut required_errors = Vec::new(); + for (index, target) in targets.iter().enumerate() { + let (key_store, cache_path) = if index == 0 && !uses_explicit_clusters { + (primary_key_store.clone(), PathBuf::from(GATEWAY_CACHE_PATH)) + } else { + self.key_store_for_additional_cluster(&target.name, &primary_key_store)? + }; + if let Err(err) = key_store.save_to(&cache_path) { + warn!(cluster = %target.name, "failed to save gateway cluster cache: {err:?}"); + } + + let mut first_error = None; + let mut response = None; + for url in &target.urls { + match self + .register_cvm(url, &key_store, &self.keys.gateway_app_id) + .await + { + Ok(value) => { + response = Some(value); + break; } Err(err) => { - warn!("Failed to register CVM: {err:?}, retrying with next dstack-gateway"); - if i == 0 { - error = err; + warn!(cluster = %target.name, %url, "Failed to register CVM: {err:?}"); + if first_error.is_none() { + first_error = Some(err); } } } } - return Err(error).context("Failed to register CVM, all dstack-gateway urls are down"); - }; - let mut wg_info = response.wg.context("Missing wg info")?; + match response { + Some(response) => prepared.push(PreparedGatewayCluster { + name: target.name.clone(), + index, + key_store, + response, + }), + None if target.required => required_errors.push(format!( + "{}: {:#}", + target.name, + first_error.unwrap_or_else(|| anyhow!("no gateway URLs configured")) + )), + None => warn!(cluster = %target.name, "optional gateway cluster is unavailable"), + } + } + if !required_errors.is_empty() { + bail!( + "Failed to register required gateway clusters: {}", + required_errors.join("; ") + ); + } - let client_ip = &wg_info.client_ip; + validate_disjoint_wireguard_addresses(&prepared)?; + for cluster in prepared { + self.apply_wireguard(cluster, force)?; + } + Ok(()) + } - // Sort peers by public key for consistent config generation - wg_info.servers.sort_by(|a, b| a.pk.cmp(&b.pk)); + fn gateway_targets(&self) -> Result> { + let targets = if self.shared.sys_config.gateway_clusters.is_empty() { + vec![GatewayTarget { + name: "default".to_string(), + urls: self.shared.sys_config.gateway_urls.clone(), + required: true, + }] + } else { + self.shared + .sys_config + .gateway_clusters + .iter() + .map(|cluster| GatewayTarget { + name: cluster.name.clone(), + urls: cluster.urls.clone(), + required: cluster.required, + }) + .collect() + }; + let mut names = std::collections::HashSet::new(); + for target in &targets { + if target.name.is_empty() + || !target + .name + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') + { + bail!("Invalid gateway cluster name: {}", target.name); + } + if !names.insert(target.name.as_str()) { + bail!("Duplicate gateway cluster name: {}", target.name); + } + if target.urls.is_empty() { + bail!("Gateway cluster {} has no URLs", target.name); + } + } + Ok(targets) + } - // Create WireGuard config - let wg_listen_port = "9182"; + fn apply_wireguard(&self, mut cluster: PreparedGatewayCluster, force: bool) -> Result<()> { + let interface = format!("dstack-wg{}", cluster.index); + let config_path = format!("/etc/wireguard/{interface}.conf"); + let listen_port = 9182_u16 + .checked_add( + cluster + .index + .try_into() + .context("too many gateway clusters")?, + ) + .context("too many gateway clusters")?; + let mut wg_info = cluster.response.wg.take().context("Missing wg info")?; + wg_info.servers.sort_by(|a, b| a.pk.cmp(&b.pk)); let mut new_config = format!( - "[Interface]\n\ - PrivateKey = {}\n\ - ListenPort = {wg_listen_port}\n\ - Address = {client_ip}/32\n\n", - key_store.wg_sk + "[Interface]\nPrivateKey = {}\nListenPort = {listen_port}\nAddress = {}/32\n\n", + cluster.key_store.wg_sk, wg_info.client_ip ); for WireGuardPeer { pk, ip, endpoint } in &wg_info.servers { let ip = ip.split('/').next().unwrap_or_default(); new_config.push_str(&format!( - "[Peer]\n\ - PublicKey = {pk}\n\ - AllowedIPs = {ip}/32\n\ - Endpoint = {endpoint}\n\ - PersistentKeepalive = 25\n", + "[Peer]\nPublicKey = {pk}\nAllowedIPs = {ip}/32\nEndpoint = {endpoint}\nPersistentKeepalive = 25\n" )); } - - // Check if config has changed (skip check if force is set) - if !force { - let current_config = fs::read_to_string(WG_CONFIG_PATH).ok(); - if current_config.as_ref() == Some(&new_config) { - info!("WireGuard config unchanged, skipping reconfiguration"); - return Ok(()); - } - } - - // The config file is also the "already applied" marker the check above - // reads, so it must not outlive a failed apply: the rules and the - // interface are what it stands for, and leaving it behind makes every - // later refresh take the early return and skip the setup it never - // finished. Nothing else repairs that -- the refresher only re-applies - // when the rendered config differs -- so the CVM would keep running with - // a half-built DSTACK_WG chain until the gateway list happened to change. - // Removing it is safe because it is rewritten before the `wg-quick down` - // below on the next attempt, and correct because /etc is a tmpfs overlay - // (see mount_overlay in dstack-prepare.sh), so the file already shares - // the reboot lifetime of the iptables rules rather than outliving them. - let applied = self - .apply_wg_config(&new_config, &wg_info, wg_listen_port) - .await; - if applied.is_err() { - if let Err(err) = fs::remove_file(WG_CONFIG_PATH) { - warn!("failed to drop the partially applied WireGuard config: {err:?}"); - } + if !force && fs::read_to_string(&config_path).ok().as_ref() == Some(&new_config) { + info!(cluster = %cluster.name, "WireGuard config unchanged"); + return Ok(()); } - applied - } - - /// Write the rendered WireGuard config, program the DSTACK_WG chain that - /// restricts its listen port to the gateway peers, and bring the interface - /// up. - /// - /// Best-effort: an error can leave the chain partly built and the jump from - /// INPUT already installed. What the caller guarantees is not that the - /// kernel state is untouched, but that the next refresh will redo the whole - /// sequence -- it removes the config file this wrote, so the "config - /// unchanged" early return cannot skip the retry. - async fn apply_wg_config( - &self, - new_config: &str, - wg_info: &WireGuardConfig, - wg_listen_port: &str, - ) -> Result<()> { - safe_write_with_mode(WG_CONFIG_PATH, new_config, 0o600) + safe_write_with_mode(&config_path, &new_config, 0o600) .context("Failed to write WireGuard config")?; + cmd!(ignore wg-quick down $interface)?; - cmd! { - ignore wg-quick down dstack-wg0; - }?; - - // Every rule change takes /run/xtables.lock, and dockerd rewrites its own - // rules on each container start/stop -- which happens throughout boot, - // exactly when this runs. Without -w, iptables does not wait for the lock; - // it exits immediately with "another app is currently holding the xtables - // lock", aborting this sequence part-way. - // - // The bound is per invocation, not for the sequence: with four fixed - // calls plus one per peer, a lock held throughout could cost roughly - // 5s * (4 + peers). That is deliberate -- it stays well inside the - // gateway checker's 600s WatchdogSec, whose unit comment already - // accounts for blocking iptables shell-outs, while still bounding each - // wait so a wedged holder cannot stall the caller indefinitely the way - // a bare -w would. - let xtables_wait = "5"; - - // Setup WireGuard iptables rules - cmd! { - // Create the chain if it doesn't exist - ignore iptables -w $xtables_wait -N DSTACK_WG 2>/dev/null; - // Flush the chain - iptables -w $xtables_wait -F DSTACK_WG; - // Remove any existing jump rule - ignore iptables -w $xtables_wait -D INPUT -p udp --dport $wg_listen_port -j DSTACK_WG 2>/dev/null; - // Insert the new jump rule at the beginning of the INPUT chain - iptables -w $xtables_wait -I INPUT -p udp --dport $wg_listen_port -j DSTACK_WG - }?; - + let chain = format!("DSTACK_WG{}", cluster.index); + cmd!(ignore iptables -N $chain 2>/dev/null)?; + cmd!(iptables -F $chain)?; + cmd!(ignore iptables -D INPUT -p udp --dport $listen_port -j $chain 2>/dev/null)?; + cmd!(iptables -I INPUT -p udp --dport $listen_port -j $chain)?; for peer in &wg_info.servers { - // Avoid issues with field-access in the macro by binding the IP to a local variable. let endpoint_ip = peer .endpoint - .split(':') - .next() + .rsplit_once(':') + .map(|(host, _)| host.trim_matches(['[', ']'])) .context("Invalid wireguard endpoint")?; - cmd!(iptables -w $xtables_wait -A DSTACK_WG -s $endpoint_ip -j ACCEPT)?; + cmd!(iptables -A $chain -s $endpoint_ip -j ACCEPT)?; } - - // Drop any UDP packets that don't come from an allowed IP. - cmd!(iptables -w $xtables_wait -A DSTACK_WG -j DROP)?; - - info!("Starting WireGuard"); - cmd!(wg-quick up dstack-wg0)?; + cmd!(iptables -A $chain -j DROP)?; + info!(cluster = %cluster.name, %interface, "Starting WireGuard"); + cmd!(wg-quick up $interface)?; Ok(()) } } @@ -2039,7 +2106,9 @@ impl GatewayRefresher { if self.keys.gateway_app_id.is_empty() { bail!("Missing allowed dstack-gateway app id"); } - if self.shared.sys_config.gateway_urls.is_empty() { + if self.shared.sys_config.gateway_urls.is_empty() + && self.shared.sys_config.gateway_clusters.is_empty() + { bail!("Missing gateway urls"); } Ok(()) @@ -3649,7 +3718,11 @@ mod kms_provider_inventory_tests { #[cfg(test)] mod gateway_registration_refresh_tests { - use super::{gateway_rpc_url, GatewayKeyStore}; + use super::{ + gateway_rpc_url, validate_disjoint_wireguard_addresses, GatewayKeyStore, + PreparedGatewayCluster, + }; + use dstack_gateway_rpc::{RegisterCvmResponse, WireGuardConfig, WireGuardPeer}; use std::os::unix::fs::PermissionsExt as _; fn key_store(cert_not_after: u64) -> GatewayKeyStore { @@ -3716,4 +3789,44 @@ mod gateway_registration_refresh_tests { assert!(!key_store(1_600).is_cert_valid_at(1_000)); assert!(!key_store(u64::MAX).is_cert_valid_at(u64::MAX)); } + + fn prepared( + name: &str, + index: usize, + client_ip: &str, + server_ip: &str, + ) -> PreparedGatewayCluster { + PreparedGatewayCluster { + name: name.into(), + index, + key_store: key_store(10_000), + response: RegisterCvmResponse { + wg: Some(WireGuardConfig { + client_ip: client_ip.into(), + servers: vec![WireGuardPeer { + pk: format!("key-{name}"), + ip: server_ip.into(), + endpoint: "192.0.2.1:51820".into(), + }], + }), + ..Default::default() + }, + } + } + + #[test] + fn independent_clusters_require_disjoint_wireguard_addresses() { + let valid = [ + prepared("primary", 0, "10.0.0.2", "10.0.0.1/32"), + prepared("secondary", 1, "10.1.0.2", "10.1.0.1/32"), + ]; + validate_disjoint_wireguard_addresses(&valid).unwrap(); + + let overlapping = [ + prepared("primary", 0, "10.0.0.2", "10.0.0.1"), + prepared("secondary", 1, "10.1.0.2", "10.0.0.1/32"), + ]; + let error = validate_disjoint_wireguard_addresses(&overlapping).unwrap_err(); + assert!(error.to_string().contains("overlapping WireGuard address")); + } } diff --git a/dstack/guest-agent/src/guest_api_service.rs b/dstack/guest-agent/src/guest_api_service.rs index 8306a104b..a625aff00 100644 --- a/dstack/guest-agent/src/guest_api_service.rs +++ b/dstack/guest-agent/src/guest_api_service.rs @@ -145,11 +145,11 @@ fn get_interfaces() -> Vec { sysinfo::Networks::new_with_refreshed_list() .into_iter() .filter_map(|(interface_name, network)| { - if !(interface_name == "dstack-wg0" + if !(interface_name.starts_with("dstack-wg") || interface_name.starts_with("enp") || interface_name.starts_with("eth")) { - // We only get dstack-wg0, enp and eth interfaces. + // We only get dstack gateway, enp and eth interfaces. // Docker bridge is not included due to privacy concerns. return None; } diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ec2164e6a..bb4af6ce7 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1562,6 +1562,7 @@ pub(crate) fn make_sys_config( } else { manifest.gateway_urls.clone() }; + let gateway_clusters = cfg.cvm.gateway_clusters.clone(); if img_ver < (0, 5, 0) { bail!("Unsupported image version: {img_ver:?}"); } @@ -1577,6 +1578,7 @@ pub(crate) fn make_sys_config( let mut sys_config = json!({ "kms_urls": kms_urls, "gateway_urls": gateway_urls, + "gateway_clusters": gateway_clusters, "pccs_url": cfg.cvm.pccs_url, "collateral_urls": { "pccs": cfg.cvm.pccs_url }, "nvidia_attestation_proxy_url": cfg.cvm.nvidia_attestation_proxy_url, diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index edf99a07c..0c356b30f 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -305,6 +305,10 @@ pub struct CvmConfig { /// The URL of the dstack-gateway server #[serde(alias = "tproxy_urls")] pub gateway_urls: Vec, + /// Independently operated gateway clusters. URLs in each entry are + /// failover endpoints for that cluster. + #[serde(default)] + pub gateway_clusters: Vec, /// The URL of the PCCS server #[serde(default)] pub pccs_url: String, @@ -704,6 +708,31 @@ impl Config { validate_http_url(name, value)?; } } + let mut cluster_names = std::collections::HashSet::new(); + for cluster in &self.cvm.gateway_clusters { + anyhow::ensure!( + !cluster.name.is_empty() + && cluster + .name + .bytes() + .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_'), + "invalid cvm.gateway_clusters name: {}", + cluster.name + ); + anyhow::ensure!( + cluster_names.insert(cluster.name.as_str()), + "duplicate cvm.gateway_clusters name: {}", + cluster.name + ); + anyhow::ensure!( + !cluster.urls.is_empty(), + "cvm.gateway_clusters.{} must contain at least one URL", + cluster.name + ); + for url in &cluster.urls { + validate_http_url("cvm.gateway_clusters.urls", url)?; + } + } for (name, value) in [ ("cvm.pccs_url", Some(self.cvm.pccs_url.as_str())), ( From c1a9501a6a31f3890f2bf6265d66ab6a149c9a92 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 20:38:12 -0700 Subject: [PATCH 2/4] fix(guest): refresh gateway clusters independently --- docs/deployment.md | 7 +- dstack/dstack-types/src/lib.rs | 3 - dstack/dstack-util/src/system_setup.rs | 124 ++++++------------------- 3 files changed, 33 insertions(+), 101 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 964dab30b..6d8018aca 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -327,6 +327,10 @@ cluster gets a separate WireGuard interface and key pair; their WireGuard address ranges must not overlap. All clusters must run the gateway app identity authorized by the CVM's KMS-issued app keys. +Clusters refresh independently. If one cluster is unavailable, its last +working WireGuard configuration remains active while other clusters continue +to register and update normally. + ```toml [cvm] kms_urls = ["https://kms.example.com:9201"] @@ -337,12 +341,9 @@ urls = [ "https://gateway-a.example.com:9202", "https://gateway-b.example.com:9202", ] -required = true - [[cvm.gateway_clusters]] name = "secondary" urls = ["https://gateway-c.example.com:9202"] -required = false ``` Restart dstack-vmm to apply changes. diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 1fce0d808..465ba815b 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -1129,9 +1129,6 @@ pub struct GatewayClusterConfig { pub name: String, /// Failover RPC endpoints belonging to this cluster. pub urls: Vec, - /// Whether failure to register this cluster makes the refresh fail. - #[serde(default = "default_true")] - pub required: bool, } #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)] diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index ea4b6d5fc..c041ed8a0 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -383,7 +383,6 @@ fn gateway_rpc_url(base: &str) -> String { struct GatewayTarget { name: String, urls: Vec, - required: bool, } struct PreparedGatewayCluster { @@ -393,27 +392,6 @@ struct PreparedGatewayCluster { response: RegisterCvmResponse, } -fn validate_disjoint_wireguard_addresses(clusters: &[PreparedGatewayCluster]) -> Result<()> { - let mut addresses = std::collections::HashMap::::new(); - for cluster in clusters { - let wg = cluster.response.wg.as_ref().context("Missing wg info")?; - for address in std::iter::once(wg.client_ip.as_str()) - .chain(wg.servers.iter().map(|peer| peer.ip.as_str())) - { - let address = address.split('/').next().unwrap_or_default().to_string(); - if let Some(other) = addresses.insert(address.clone(), &cluster.name) { - bail!( - "Gateway clusters {} and {} use overlapping WireGuard address {}", - other, - cluster.name, - address - ); - } - } - } - Ok(()) -} - struct GatewayContext<'a> { shared: &'a HostShared, keys: &'a AppKeys, @@ -644,13 +622,19 @@ impl<'a> GatewayContext<'a> { warn!("failed to save gateway cache: {err:?}"); } - let mut prepared = Vec::new(); - let mut required_errors = Vec::new(); + let mut errors = Vec::new(); for (index, target) in targets.iter().enumerate() { - let (key_store, cache_path) = if index == 0 && !uses_explicit_clusters { - (primary_key_store.clone(), PathBuf::from(GATEWAY_CACHE_PATH)) + let key_store_result = if index == 0 && !uses_explicit_clusters { + Ok((primary_key_store.clone(), PathBuf::from(GATEWAY_CACHE_PATH))) } else { - self.key_store_for_additional_cluster(&target.name, &primary_key_store)? + self.key_store_for_additional_cluster(&target.name, &primary_key_store) + }; + let (key_store, cache_path) = match key_store_result { + Ok(value) => value, + Err(err) => { + errors.push(format!("{}: {err:#}", target.name)); + continue; + } }; if let Err(err) = key_store.save_to(&cache_path) { warn!(cluster = %target.name, "failed to save gateway cluster cache: {err:?}"); @@ -675,33 +659,29 @@ impl<'a> GatewayContext<'a> { } } } - match response { - Some(response) => prepared.push(PreparedGatewayCluster { - name: target.name.clone(), - index, - key_store, - response, - }), - None if target.required => required_errors.push(format!( + let Some(response) = response else { + errors.push(format!( "{}: {:#}", target.name, first_error.unwrap_or_else(|| anyhow!("no gateway URLs configured")) - )), - None => warn!(cluster = %target.name, "optional gateway cluster is unavailable"), + )); + continue; + }; + let cluster = PreparedGatewayCluster { + name: target.name.clone(), + index, + key_store, + response, + }; + if let Err(err) = self.apply_wireguard(cluster, force) { + errors.push(format!("{}: {err:#}", target.name)); } } - if !required_errors.is_empty() { - bail!( - "Failed to register required gateway clusters: {}", - required_errors.join("; ") - ); - } - - validate_disjoint_wireguard_addresses(&prepared)?; - for cluster in prepared { - self.apply_wireguard(cluster, force)?; + if errors.is_empty() { + Ok(()) + } else { + bail!("Failed to refresh gateway clusters: {}", errors.join("; ")) } - Ok(()) } fn gateway_targets(&self) -> Result> { @@ -709,7 +689,6 @@ impl<'a> GatewayContext<'a> { vec![GatewayTarget { name: "default".to_string(), urls: self.shared.sys_config.gateway_urls.clone(), - required: true, }] } else { self.shared @@ -719,7 +698,6 @@ impl<'a> GatewayContext<'a> { .map(|cluster| GatewayTarget { name: cluster.name.clone(), urls: cluster.urls.clone(), - required: cluster.required, }) .collect() }; @@ -3718,11 +3696,7 @@ mod kms_provider_inventory_tests { #[cfg(test)] mod gateway_registration_refresh_tests { - use super::{ - gateway_rpc_url, validate_disjoint_wireguard_addresses, GatewayKeyStore, - PreparedGatewayCluster, - }; - use dstack_gateway_rpc::{RegisterCvmResponse, WireGuardConfig, WireGuardPeer}; + use super::{gateway_rpc_url, GatewayKeyStore}; use std::os::unix::fs::PermissionsExt as _; fn key_store(cert_not_after: u64) -> GatewayKeyStore { @@ -3789,44 +3763,4 @@ mod gateway_registration_refresh_tests { assert!(!key_store(1_600).is_cert_valid_at(1_000)); assert!(!key_store(u64::MAX).is_cert_valid_at(u64::MAX)); } - - fn prepared( - name: &str, - index: usize, - client_ip: &str, - server_ip: &str, - ) -> PreparedGatewayCluster { - PreparedGatewayCluster { - name: name.into(), - index, - key_store: key_store(10_000), - response: RegisterCvmResponse { - wg: Some(WireGuardConfig { - client_ip: client_ip.into(), - servers: vec![WireGuardPeer { - pk: format!("key-{name}"), - ip: server_ip.into(), - endpoint: "192.0.2.1:51820".into(), - }], - }), - ..Default::default() - }, - } - } - - #[test] - fn independent_clusters_require_disjoint_wireguard_addresses() { - let valid = [ - prepared("primary", 0, "10.0.0.2", "10.0.0.1/32"), - prepared("secondary", 1, "10.1.0.2", "10.1.0.1/32"), - ]; - validate_disjoint_wireguard_addresses(&valid).unwrap(); - - let overlapping = [ - prepared("primary", 0, "10.0.0.2", "10.0.0.1"), - prepared("secondary", 1, "10.1.0.2", "10.0.0.1/32"), - ]; - let error = validate_disjoint_wireguard_addresses(&overlapping).unwrap_err(); - assert!(error.to_string().contains("overlapping WireGuard address")); - } } From 5b4d21d0a7fab50c793c75122a1d05bdfda3b918 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 20:50:11 -0700 Subject: [PATCH 3/4] fix(guest): recover failed gateway config applies --- dstack/dstack-util/src/system_setup.rs | 140 ++++++++++++++++++++----- 1 file changed, 116 insertions(+), 24 deletions(-) diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index c041ed8a0..6df4316f1 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -392,6 +392,30 @@ struct PreparedGatewayCluster { response: RegisterCvmResponse, } +fn wireguard_endpoint_hosts(config: &str) -> Result> { + config + .lines() + .filter_map(|line| line.trim().strip_prefix("Endpoint = ")) + .map(|endpoint| { + endpoint + .rsplit_once(':') + .map(|(host, _)| host.trim_matches(['[', ']']).to_string()) + .context("invalid WireGuard endpoint") + }) + .collect() +} + +fn remove_partial_wireguard_config(path: &str, cluster: &str) { + if let Err(error) = fs::remove_file(path) { + if error.kind() != std::io::ErrorKind::NotFound { + warn!( + cluster = cluster, + "failed to remove partially applied WireGuard config: {error}" + ); + } + } +} + struct GatewayContext<'a> { shared: &'a HostShared, keys: &'a AppKeys, @@ -680,7 +704,7 @@ impl<'a> GatewayContext<'a> { if errors.is_empty() { Ok(()) } else { - bail!("Failed to refresh gateway clusters: {}", errors.join("; ")) + bail!("failed to refresh gateway clusters: {}", errors.join("; ")) } } @@ -709,13 +733,13 @@ impl<'a> GatewayContext<'a> { .bytes() .all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_') { - bail!("Invalid gateway cluster name: {}", target.name); + bail!("invalid gateway cluster name: {}", target.name); } if !names.insert(target.name.as_str()) { - bail!("Duplicate gateway cluster name: {}", target.name); + bail!("duplicate gateway cluster name: {}", target.name); } if target.urls.is_empty() { - bail!("Gateway cluster {} has no URLs", target.name); + bail!("gateway cluster {} has no URLs", target.name); } } Ok(targets) @@ -732,7 +756,7 @@ impl<'a> GatewayContext<'a> { .context("too many gateway clusters")?, ) .context("too many gateway clusters")?; - let mut wg_info = cluster.response.wg.take().context("Missing wg info")?; + let mut wg_info = cluster.response.wg.take().context("missing wg info")?; wg_info.servers.sort_by(|a, b| a.pk.cmp(&b.pk)); let mut new_config = format!( "[Interface]\nPrivateKey = {}\nListenPort = {listen_port}\nAddress = {}/32\n\n", @@ -744,29 +768,83 @@ impl<'a> GatewayContext<'a> { "[Peer]\nPublicKey = {pk}\nAllowedIPs = {ip}/32\nEndpoint = {endpoint}\nPersistentKeepalive = 25\n" )); } - if !force && fs::read_to_string(&config_path).ok().as_ref() == Some(&new_config) { + let old_config = fs::read_to_string(&config_path).ok(); + if !force && old_config.as_ref() == Some(&new_config) { info!(cluster = %cluster.name, "WireGuard config unchanged"); return Ok(()); } - safe_write_with_mode(&config_path, &new_config, 0o600) - .context("Failed to write WireGuard config")?; + let new_endpoints = wireguard_endpoint_hosts(&new_config)?; + let applied = self.apply_wireguard_config( + &interface, + &config_path, + listen_port, + &new_config, + &new_endpoints, + ); + if let Err(error) = applied { + // Registration and refresh are independent per cluster. Preserve + // this cluster's last-known-good config if applying its replacement + // fails; a cluster with no prior config removes the false marker so + // the checker takes its fast missing-config retry path. + if let Some(old_config) = old_config { + match wireguard_endpoint_hosts(&old_config).and_then(|endpoints| { + self.apply_wireguard_config( + &interface, + &config_path, + listen_port, + &old_config, + &endpoints, + ) + }) { + Ok(()) => { + warn!(cluster = %cluster.name, "restored previous WireGuard config after refresh failure") + } + Err(rollback_error) => { + warn!(cluster = %cluster.name, "failed to restore previous WireGuard config: {rollback_error:#}"); + remove_partial_wireguard_config(&config_path, &cluster.name); + } + } + } else { + remove_partial_wireguard_config(&config_path, &cluster.name); + } + return Err(error); + } + Ok(()) + } + + fn apply_wireguard_config( + &self, + interface: &str, + config_path: &str, + listen_port: u16, + config: &str, + endpoint_hosts: &[String], + ) -> Result<()> { + safe_write_with_mode(config_path, config, 0o600) + .context("failed to write WireGuard config")?; cmd!(ignore wg-quick down $interface)?; - let chain = format!("DSTACK_WG{}", cluster.index); - cmd!(ignore iptables -N $chain 2>/dev/null)?; - cmd!(iptables -F $chain)?; - cmd!(ignore iptables -D INPUT -p udp --dport $listen_port -j $chain 2>/dev/null)?; - cmd!(iptables -I INPUT -p udp --dport $listen_port -j $chain)?; - for peer in &wg_info.servers { - let endpoint_ip = peer - .endpoint - .rsplit_once(':') - .map(|(host, _)| host.trim_matches(['[', ']'])) - .context("Invalid wireguard endpoint")?; - cmd!(iptables -A $chain -s $endpoint_ip -j ACCEPT)?; - } - cmd!(iptables -A $chain -j DROP)?; - info!(cluster = %cluster.name, %interface, "Starting WireGuard"); + // Docker also updates iptables throughout boot. Bound every lock wait + // so a transient xtables.lock holder neither breaks the cluster update + // nor stalls the checker indefinitely. + let xtables_wait = "5"; + let chain = format!("DSTACK_WG{}", interface.trim_start_matches("dstack-wg")); + if interface == "dstack-wg0" { + // Remove the pre-multi-cluster chain after upgrading. The new + // per-interface chain below owns the same listen port. + cmd!(ignore iptables -w $xtables_wait -D INPUT -p udp --dport $listen_port -j DSTACK_WG 2>/dev/null)?; + cmd!(ignore iptables -w $xtables_wait -F DSTACK_WG 2>/dev/null)?; + cmd!(ignore iptables -w $xtables_wait -X DSTACK_WG 2>/dev/null)?; + } + cmd!(ignore iptables -w $xtables_wait -N $chain 2>/dev/null)?; + cmd!(iptables -w $xtables_wait -F $chain)?; + cmd!(ignore iptables -w $xtables_wait -D INPUT -p udp --dport $listen_port -j $chain 2>/dev/null)?; + cmd!(iptables -w $xtables_wait -I INPUT -p udp --dport $listen_port -j $chain)?; + for endpoint_host in endpoint_hosts { + cmd!(iptables -w $xtables_wait -A $chain -s $endpoint_host -j ACCEPT)?; + } + cmd!(iptables -w $xtables_wait -A $chain -j DROP)?; + info!(%interface, "starting WireGuard"); cmd!(wg-quick up $interface)?; Ok(()) } @@ -3696,7 +3774,7 @@ mod kms_provider_inventory_tests { #[cfg(test)] mod gateway_registration_refresh_tests { - use super::{gateway_rpc_url, GatewayKeyStore}; + use super::{gateway_rpc_url, wireguard_endpoint_hosts, GatewayKeyStore}; use std::os::unix::fs::PermissionsExt as _; fn key_store(cert_not_after: u64) -> GatewayKeyStore { @@ -3763,4 +3841,18 @@ mod gateway_registration_refresh_tests { assert!(!key_store(1_600).is_cert_valid_at(1_000)); assert!(!key_store(u64::MAX).is_cert_valid_at(u64::MAX)); } + + #[test] + fn wireguard_endpoint_hosts_support_dns_ipv4_and_ipv6() { + let config = r#" +Endpoint = gateway.example.com:51820 +Endpoint = 192.0.2.1:51821 +Endpoint = [2001:db8::1]:51822 +"#; + assert_eq!( + wireguard_endpoint_hosts(config).unwrap(), + ["gateway.example.com", "192.0.2.1", "2001:db8::1"] + ); + assert!(wireguard_endpoint_hosts("Endpoint = missing-port").is_err()); + } } From 7d20a1e87d0b4c1e64a80603ce76e57038ae632a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 16 Aug 2026 21:16:45 -0700 Subject: [PATCH 4/4] fix(config): reject mixed gateway syntax --- docs/deployment.md | 5 +++++ dstack/dstack-util/src/system_setup.rs | 5 +++++ dstack/vmm/src/config.rs | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/docs/deployment.md b/docs/deployment.md index 6d8018aca..dec73fe4b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -331,6 +331,11 @@ Clusters refresh independently. If one cluster is unavailable, its last working WireGuard configuration remains active while other clusters continue to register and update normally. +`gateway_urls` and `gateway_clusters` are mutually exclusive in the VMM +configuration. The VMM refuses to start if both are non-empty. For compatibility +with sys-config files produced elsewhere, the guest prefers `gateway_clusters` +and logs a warning when both forms are present. + ```toml [cvm] kms_urls = ["https://kms.example.com:9201"] diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 6df4316f1..0978a6f28 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -709,6 +709,11 @@ impl<'a> GatewayContext<'a> { } fn gateway_targets(&self) -> Result> { + if !self.shared.sys_config.gateway_urls.is_empty() + && !self.shared.sys_config.gateway_clusters.is_empty() + { + warn!("both gateway_urls and gateway_clusters are configured; ignoring gateway_urls"); + } let targets = if self.shared.sys_config.gateway_clusters.is_empty() { vec![GatewayTarget { name: "default".to_string(), diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 0c356b30f..e13464af8 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -708,6 +708,10 @@ impl Config { validate_http_url(name, value)?; } } + anyhow::ensure!( + self.cvm.gateway_urls.is_empty() || self.cvm.gateway_clusters.is_empty(), + "cvm.gateway_urls and cvm.gateway_clusters cannot both be configured" + ); let mut cluster_names = std::collections::HashSet::new(); for cluster in &self.cvm.gateway_clusters { anyhow::ensure!( @@ -1187,6 +1191,20 @@ mod tests { assert!(format!("{:#}", config.validate().unwrap_err()).contains("invalid CID")); } + #[test] + fn config_validation_rejects_mixed_gateway_syntax() { + let mut config = default_config(); + config.cvm.gateway_urls = vec!["https://legacy-gateway.example.com".into()]; + config.cvm.gateway_clusters = vec![dstack_types::GatewayClusterConfig { + name: "primary".into(), + urls: vec!["https://gateway.example.com".into()], + }]; + let error = config.validate().unwrap_err(); + assert!(error + .to_string() + .contains("gateway_urls and cvm.gateway_clusters cannot both")); + } + #[test] fn config_validation_does_not_require_supervisor_startup_paths_when_disabled() { let mut config = default_config();