feat(guest): support multiple gateway clusters

docs/deployment.md

@@ -321,6 +321,36 @@ kms_urls = ["https://kms.example.com:9201"]
 gateway_urls = ["https://gateway.example.com:9202"]
 ```
 
+`gateway_urls` is a failover list for one gateway cluster. To register a CVM
+with independently operated clusters, configure explicit groups instead. Each
+cluster gets a separate WireGuard interface and key pair; their WireGuard
+address ranges must not overlap. All clusters must run the gateway app identity
+authorized by the CVM's KMS-issued app keys.
+
+Clusters refresh independently. If one cluster is unavailable, its last
+working WireGuard configuration remains active while other clusters continue
+to register and update normally.
+
+`gateway_urls` and `gateway_clusters` are mutually exclusive in the VMM
+configuration. The VMM refuses to start if both are non-empty. For compatibility
+with sys-config files produced elsewhere, the guest prefers `gateway_clusters`
+and logs a warning when both forms are present.
+
+```toml
+[cvm]
+kms_urls = ["https://kms.example.com:9201"]
+
+[[cvm.gateway_clusters]]
+name = "primary"
+urls = [
+  "https://gateway-a.example.com:9202",
+  "https://gateway-b.example.com:9202",
+]
+[[cvm.gateway_clusters]]
+name = "secondary"
+urls = ["https://gateway-c.example.com:9202"]
+```
+
 Restart dstack-vmm to apply changes.
 
 ---

dstack/dstack-types/src/lib.rs

@@ -1093,6 +1093,11 @@ pub struct SysConfig {
     pub kms_urls: Vec<String>,
     #[serde(default, alias = "tproxy_urls")]
     pub gateway_urls: Vec<String>,
+    /// Independently operated gateway clusters. URLs within one entry are
+    /// failover endpoints for the same cluster. When empty, `gateway_urls` is
+    /// treated as one legacy cluster.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub gateway_clusters: Vec<GatewayClusterConfig>,
     /// Backward-compatible input for sys-config files produced by older hosts.
     #[serde(default, rename = "pccs_url", skip_serializing)]
     legacy_pccs_url: Option<String>,
@@ -1117,6 +1122,15 @@ pub struct SysConfig {
     pub vm_config: String,
 }
 
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct GatewayClusterConfig {
+    /// Stable local name used for the per-cluster key cache.
+    pub name: String,
+    /// Failover RPC endpoints belonging to this cluster.
+    pub urls: Vec<String>,
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub struct CollateralUrls {

dstack/dstack-util/src/gateway_checker.rs

@@ -28,15 +28,15 @@
 //! unit tested without a gateway, a KMS, or a WireGuard interface; all I/O
 //! lives in [`cmd_gateway_checker`].
 
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{Context, Result};
 use cmd_lib::run_fun as cmd;
 use sd_notify::NotifyState;
 use tracing::{error, info, warn};
 
-use crate::system_setup::{GatewayRefresher, WG_CONFIG_PATH, WG_INTERFACE};
+use crate::system_setup::GatewayRefresher;
 
 /// How often the loop samples the world.
 const POLL_INTERVAL: Duration = Duration::from_secs(10);
@@ -104,7 +104,7 @@ impl Backoff {
 struct Observation {
     /// Seconds since the UNIX epoch.
     now: i64,
-    /// Whether `/etc/wireguard/dstack-wg0.conf` exists.
+    /// Whether at least one `/etc/wireguard/dstack-wg*.conf` exists.
     config_present: bool,
     /// Most recent handshake as a UNIX timestamp; `None` if the interface has
     /// never completed one (or does not exist yet).
@@ -258,12 +258,28 @@ fn parse_latest_handshake(output: &str) -> Option<i64> {
         .max()
 }
 
-fn wg_config_present() -> bool {
-    Path::new(WG_CONFIG_PATH).exists()
+fn configured_gateway_interfaces() -> Vec<String> {
+    let Ok(entries) = std::fs::read_dir("/etc/wireguard") else {
+        return Vec::new();
+    };
+    entries
+        .filter_map(Result::ok)
+        .filter_map(|entry| entry.file_name().into_string().ok())
+        .filter_map(|name| {
+            name.strip_suffix(".conf")
+                .filter(|name| name.starts_with("dstack-wg"))
+                .map(str::to_string)
+        })
+        .collect()
 }
 
 fn observe(now: i64) -> Observation {
-    let config_present = wg_config_present();
+    let interfaces = configured_gateway_interfaces();
+    let config_present = !interfaces.is_empty();
+    let handshakes = interfaces
+        .iter()
+        .map(|interface| latest_handshake(interface))
+        .collect::<Option<Vec<_>>>();
     Observation {
         now,
         config_present,
@@ -272,9 +288,10 @@ fn observe(now: i64) -> Observation {
         // the probe matters because that is precisely the state a gateway
         // outage parks the CVM in: otherwise we would fork `wg` every poll for
         // the entire outage to answer a question nobody asks.
-        latest_handshake: config_present
-            .then(|| latest_handshake(WG_INTERFACE))
-            .flatten(),
+        // The least healthy cluster drives recovery. A missing handshake on
+        // any configured interface is represented as None; otherwise the
+        // oldest cluster handshake is the staleness boundary.
+        latest_handshake: handshakes.and_then(|values| values.into_iter().min()),
     }
 }
 
@@ -349,7 +366,7 @@ pub async fn cmd_gateway_checker(args: GatewayCheckerArgs) -> Result<()> {
     // has been told, which is state this loop should not have to carry. The
     // consequence is that a boot error stays on the VMM after the checker
     // recovers, until the VM restarts.
-    let mut checker = Checker::starting(now_secs(), wg_config_present());
+    let mut checker = Checker::starting(now_secs(), !configured_gateway_interfaces().is_empty());
     loop {
         // Ping before the work, not after, so a refresh that never returns
         // stops the pings. Nothing else can do this for us: a refresh spends