From dcec2abf2cef6a77bf752ac2936d313e4e9fd485 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Mon, 3 Aug 2026 08:06:07 +0000 Subject: [PATCH 1/2] refactor(driver): extract shared supervisor binary helpers into openshell-core Move supervisor binary extraction, caching, and validation helpers from the Docker driver into openshell-core::driver_utils so both Docker and Podman drivers can reuse them. Moved helpers: extract_first_tar_entry, write_cache_binary_atomic, supervisor_cache_path, temp_extract_container_name, and validate_linux_elf_binary. The shared extract_first_tar_entry gains entry-type and empty-payload checks that the Docker-local version lacked. supervisor_cache_path takes a driver_subdir parameter so each driver caches under its own namespace (docker-supervisor vs podman-supervisor). Signed-off-by: Giuseppe Scrivano --- Cargo.lock | 1 + crates/openshell-core/Cargo.toml | 2 + crates/openshell-core/src/driver_utils.rs | 139 ++++++++++++++++++- crates/openshell-driver-docker/Cargo.toml | 4 +- crates/openshell-driver-docker/src/lib.rs | 144 ++------------------ crates/openshell-driver-docker/src/tests.rs | 13 +- 6 files changed, 160 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f3f7dcdca..31329350bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3738,6 +3738,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", + "tar", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index e138e1eee1..c12b83cf12 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -27,6 +27,8 @@ ipnet = "2" base64 = { workspace = true } chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true } reqwest = { workspace = true, features = ["blocking", "rustls-tls-native-roots"], optional = true } +tar = "0.4" +tempfile = "3" [target.'cfg(unix)'.dependencies] nix = { workspace = true } diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 9bcca9f11d..65d9641488 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -3,7 +3,7 @@ //! Utility helpers shared across compute-driver crates. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use crate::proto::compute::v1::{DriverSandbox, GetCapabilitiesResponse}; @@ -420,6 +420,143 @@ pub fn supervisor_image_should_refresh(image: &str) -> bool { matches!(supervisor_image_tag(image), Some("dev" | "latest")) } +// --------------------------------------------------------------------------- +// Supervisor binary extraction helpers (shared by Docker and Podman drivers) +// --------------------------------------------------------------------------- + +/// Extract the payload of the first regular-file entry in a tar archive. +/// +/// Container archive endpoints return a single-file tar when `path` points to +/// a file, so only the first entry is consumed. Returns an error when the +/// archive is empty, the first entry is not a regular file, or the payload is +/// empty. +pub fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { + let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); + let mut entries = archive + .entries() + .map_err(|err| format!("open tar archive: {err}"))?; + let mut entry = entries + .next() + .ok_or_else(|| "tar archive was empty".to_string())? + .map_err(|err| format!("read tar entry: {err}"))?; + let kind = entry.header().entry_type(); + if !kind.is_file() { + return Err(format!( + "expected a regular file in tar archive, got type {kind:?}" + )); + } + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut bytes) + .map_err(|err| format!("read tar entry payload: {err}"))?; + if bytes.is_empty() { + return Err("tar entry payload was empty".to_string()); + } + Ok(bytes) +} + +/// Atomically write `bytes` to `final_path` via a sibling temp file. +/// +/// Creates parent directories as needed. The temp file is synced, `chmod 755` +/// (on Unix), and renamed into place so concurrent readers never observe a +/// partial write. Returns a human-readable error string on failure. +pub fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> Result<(), String> { + let dir = final_path + .parent() + .ok_or_else(|| format!("cache path '{}' has no parent", final_path.display()))?; + std::fs::create_dir_all(dir) + .map_err(|err| format!("failed to create cache dir '{}': {err}", dir.display()))?; + + let mut temp = tempfile::Builder::new() + .prefix(".openshell-sandbox-") + .tempfile_in(dir) + .map_err(|err| format!("failed to create temp file in '{}': {err}", dir.display()))?; + std::io::Write::write_all(&mut temp, bytes) + .map_err(|err| format!("failed to write supervisor binary: {err}"))?; + temp.as_file() + .sync_all() + .map_err(|err| format!("failed to sync supervisor binary: {err}"))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)) + .map_err(|err| format!("failed to chmod supervisor binary: {err}"))?; + } + + temp.persist(final_path).map_err(|err| { + format!( + "failed to persist supervisor binary to '{}': {}", + final_path.display(), + err.error, + ) + })?; + Ok(()) +} + +/// Return the host-side cache path for an extracted supervisor binary. +/// +/// The path is `$XDG_DATA_HOME/openshell///openshell-sandbox`. +/// `driver_subdir` distinguishes caches across drivers (e.g. `"docker-supervisor"`, +/// `"podman-supervisor"`). +pub fn supervisor_cache_path(driver_subdir: &str, digest: &str) -> Result { + let base = crate::paths::xdg_data_dir() + .map_err(|err| format!("failed to resolve XDG data dir: {err}"))?; + Ok(supervisor_cache_path_with_base( + &base, + driver_subdir, + digest, + )) +} + +/// [`supervisor_cache_path`] with an explicit base directory (for testing). +pub fn supervisor_cache_path_with_base(base: &Path, driver_subdir: &str, digest: &str) -> PathBuf { + let sanitized = digest.replace(':', "-"); + base.join("openshell") + .join(driver_subdir) + .join(sanitized) + .join("openshell-sandbox") +} + +/// Generate a unique container name for supervisor binary extraction. +/// +/// Uses the process ID and an atomic counter to avoid collisions across +/// concurrent gateway starts. +pub fn temp_extract_container_name() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let pid = std::process::id(); + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + format!("openshell-supervisor-extract-{pid}-{seq}") +} + +/// Validate that the file at `path` starts with the ELF magic bytes (`\x7fELF`). +/// +/// Returns a human-readable error when the file cannot be read or is not a +/// Linux ELF binary. +pub fn validate_linux_elf_binary(path: &Path) -> Result<(), String> { + use std::io::Read; + let mut file = std::fs::File::open(path).map_err(|err| { + format!( + "failed to open supervisor binary '{}': {err}", + path.display() + ) + })?; + let mut magic = [0u8; 4]; + file.read_exact(&mut magic).map_err(|err| { + format!( + "failed to read supervisor binary '{}': {err}", + path.display() + ) + })?; + if magic != [0x7f, b'E', b'L', b'F'] { + return Err(format!( + "supervisor binary '{}' is not a Linux ELF executable", + path.display(), + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7e1bc069cb..d09edc3968 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -23,13 +23,13 @@ serde = { workspace = true } serde_json = { workspace = true } prost-types = { workspace = true } bollard = { version = "0.20" } -tar = "0.4" -tempfile = "3" url = { workspace = true } [dev-dependencies] prost-types = { workspace = true } +tar = "0.4" temp-env = "0.3" +tempfile = "3" [lints] workspace = true diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index dd4d9ef0f0..7ffde8e3da 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -26,7 +26,8 @@ use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - supervisor_image_should_refresh, + extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -53,7 +54,6 @@ use openshell_core::proto_struct::{ }; use openshell_core::{Config, Error, Result as CoreResult}; use std::collections::{HashMap, HashSet}; -use std::io::Read; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -3116,7 +3116,7 @@ fn resolve_supervisor_bin_source( // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. if let Some(path) = docker_config.supervisor_bin.clone() { let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path)?; + validate_linux_elf_binary(&path).map_err(Error::config)?; return Ok(SupervisorBinSource::Binary(path)); } @@ -3256,25 +3256,14 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core )) })?; - let cache_path = supervisor_cache_path(&digest)?; + let cache_path = + openshell_core::driver_utils::supervisor_cache_path("docker-supervisor", &digest) + .map_err(Error::config)?; if cache_path.is_file() { - validate_linux_elf_binary(&cache_path)?; + validate_linux_elf_binary(&cache_path).map_err(Error::config)?; return Ok(cache_path); } - let cache_dir = cache_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - cache_path.display(), - )) - })?; - std::fs::create_dir_all(cache_dir).map_err(|err| { - Error::config(format!( - "failed to create docker supervisor cache dir '{}': {err}", - cache_dir.display(), - )) - })?; - info!( image = image, digest = digest, @@ -3283,8 +3272,8 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core ); let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes)?; - validate_linux_elf_binary(&cache_path)?; + write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; + validate_linux_elf_binary(&cache_path).map_err(Error::config)?; Ok(cache_path) } @@ -3377,98 +3366,6 @@ async fn download_binary_from_container( }) } -/// Extract the payload of the first regular-file entry in a tar archive. -/// Docker's `/containers//archive` endpoint returns a single-file tar -/// when `path` points to a file, so we only need the first entry. -fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { - let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let mut entries = archive - .entries() - .map_err(|err| format!("open tar archive: {err}"))?; - let mut entry = entries - .next() - .ok_or_else(|| "tar archive was empty".to_string())? - .map_err(|err| format!("read tar entry: {err}"))?; - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|err| format!("read tar entry payload: {err}"))?; - Ok(bytes) -} - -fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { - let dir = final_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - final_path.display(), - )) - })?; - let mut temp = tempfile::Builder::new() - .prefix(".openshell-sandbox-") - .tempfile_in(dir) - .map_err(|err| { - Error::config(format!( - "failed to create temp file for supervisor binary in '{}': {err}", - dir.display(), - )) - })?; - std::io::Write::write_all(&mut temp, bytes).map_err(|err| { - Error::config(format!( - "failed to write supervisor binary to temp file: {err}", - )) - })?; - temp.as_file().sync_all().map_err(|err| { - Error::config(format!("failed to sync supervisor binary temp file: {err}")) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( - |err| { - Error::config(format!( - "failed to chmod supervisor binary temp file: {err}", - )) - }, - )?; - } - - temp.persist(final_path).map_err(|err| { - Error::config(format!( - "failed to rename supervisor binary into '{}': {}", - final_path.display(), - err.error, - )) - })?; - Ok(()) -} - -/// Cache path for an extracted supervisor binary, keyed by the image's -/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed -/// directory keeps stale extractions from earlier releases isolated so they -/// can be GC'd without affecting the active binary. -fn supervisor_cache_path(digest: &str) -> CoreResult { - let base = openshell_core::paths::xdg_data_dir() - .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; - Ok(supervisor_cache_path_with_base(&base, digest)) -} - -fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { - let sanitized = digest.replace(':', "-"); - base.join("openshell") - .join("docker-supervisor") - .join(sanitized) - .join("openshell-sandbox") -} - -fn temp_extract_container_name() -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static SEQ: AtomicU64 = AtomicU64::new(0); - let pid = std::process::id(); - let seq = SEQ.fetch_add(1, Ordering::Relaxed); - format!("openshell-supervisor-extract-{pid}-{seq}") -} - fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { if !path.is_file() { return Err(Error::config(format!( @@ -3484,29 +3381,6 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult CoreResult<()> { - let mut file = std::fs::File::open(path).map_err(|err| { - Error::config(format!( - "failed to open docker supervisor binary '{}': {err}", - path.display() - )) - })?; - let mut magic = [0_u8; 4]; - file.read_exact(&mut magic).map_err(|err| { - Error::config(format!( - "failed to read docker supervisor binary '{}': {err}", - path.display() - )) - })?; - if magic != [0x7f, b'E', b'L', b'F'] { - return Err(Error::config(format!( - "docker supervisor binary '{}' must be a Linux ELF executable", - path.display() - ))); - } - Ok(()) -} - fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { docker_config.guest_tls_ca.is_some() && docker_config.guest_tls_cert.is_some() diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index ac525c705c..42a93daf03 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -5,7 +5,7 @@ use super::*; use openshell_core::config::DEFAULT_SERVER_PORT; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_NAMESPACE, supervisor_cache_path_with_base, }; use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, @@ -2380,8 +2380,11 @@ fn docker_supervisor_image_refreshes_mutable_tags_only() { #[test] fn supervisor_cache_path_namespaces_by_digest_under_openshell_data_dir() { let base = PathBuf::from("/var/cache/share"); - let path = - supervisor_cache_path_with_base(&base, "sha256:abc123deadbeef0123456789cafe0123456789fe"); + let path = supervisor_cache_path_with_base( + &base, + "docker-supervisor", + "sha256:abc123deadbeef0123456789cafe0123456789fe", + ); assert_eq!( path, @@ -2394,8 +2397,8 @@ fn supervisor_cache_path_namespaces_by_digest_under_openshell_data_dir() { #[test] fn supervisor_cache_path_isolates_different_digests() { let base = PathBuf::from("/data"); - let left = supervisor_cache_path_with_base(&base, "sha256:aaaaaaaa"); - let right = supervisor_cache_path_with_base(&base, "sha256:bbbbbbbb"); + let left = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:aaaaaaaa"); + let right = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:bbbbbbbb"); assert_ne!( left.parent().unwrap(), right.parent().unwrap(), From 03b274d5ad99300294a47a78c8d83bb47195c8ec Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Mon, 3 Aug 2026 08:06:39 +0000 Subject: [PATCH 2/2] feat(driver-podman): add userns config Add a `userns` option to the Podman compute driver that maps to Podman's user namespace modes. The mode string is split on the first colon into the API's `nsmode` and `value` fields so parameterized values like `auto:size=65536` and `keep-id:uid=1000,gid=1000` are forwarded correctly. When the mode is `auto`, the container spec also sets `idmappings.AutoUserNs = true` as required by the API. An allowlist validates the mode at startup: `auto` and `keep-id` accept optional parameters; `host`, `private`, and `nomap` reject them; everything else is an error. Podman image volumes use overlay mounts internally and the kernel does not support idmapped mounts on overlay (`mount_setattr` returns EINVAL). When userns is configured (any mode except `host`), the driver extracts the supervisor binary from the image to a host-side cache and bind-mounts it instead of using an image volume. Configurable via TOML `userns = "auto"`, CLI `--userns`, or environment variable `OPENSHELL_PODMAN_USERNS`. Signed-off-by: Giuseppe Scrivano --- .../skills/debug-openshell-cluster/SKILL.md | 20 + architecture/compute-runtimes.md | 4 +- crates/openshell-core/Cargo.toml | 5 +- crates/openshell-core/src/driver_utils.rs | 2 + crates/openshell-driver-docker/Cargo.toml | 2 +- crates/openshell-driver-podman/Cargo.toml | 2 +- crates/openshell-driver-podman/README.md | 1 + crates/openshell-driver-podman/src/client.rs | 26 ++ crates/openshell-driver-podman/src/config.rs | 305 ++++++++++++++ .../openshell-driver-podman/src/container.rs | 376 +++++++++++++++++- crates/openshell-driver-podman/src/driver.rs | 249 +++++++++++- crates/openshell-driver-podman/src/main.rs | 18 + .../src/compute/driver_config.rs | 3 + docs/reference/gateway-config.mdx | 9 + e2e/rust/Cargo.lock | 19 + e2e/rust/Cargo.toml | 6 + e2e/rust/tests/podman_userns.rs | 302 ++++++++++++++ 17 files changed, 1316 insertions(+), 33 deletions(-) create mode 100644 e2e/rust/tests/podman_userns.rs diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index cbff462d45..f1463e28d6 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -216,6 +216,26 @@ Common findings: cannot bypass slirp4netns host-loopback isolation. Do not work around discovery failures by broadening the primary gateway listener to `0.0.0.0`. +When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`): + +- Supervisor delivery uses bind-mount fallback instead of image volumes because + overlay mounts do not support `idmapped` mounts. The supervisor binary is + extracted from the supervisor image and cached at + `$XDG_DATA_HOME/openshell/podman-supervisor/` (typically + `~/.local/share/openshell/podman-supervisor/`). +- Stale cache: if the supervisor image is updated but the cached binary is not + refreshed, sandbox creation may fail with an ELF validation error or version + mismatch. Remove the cache directory and retry. +- `auto` mode requires subuid/subgid ranges for the current user in + `/etc/subuid` and `/etc/subgid`. If missing, Podman returns a user-namespace + mapping error at container creation. +- `private` mode requires explicit `uidmap` and `gidmap` arrays in the TOML + config. Without both, the gateway rejects the config at startup. + Rootless Podman uses intermediate IDs (e.g. `uidmap = ["0:0:1", "1:1:65535"]`); + rootful Podman uses absolute host IDs (e.g. `uidmap = ["0:1000:1", "1:100000:65536"]`). +- `nomap` (without hyphen) is accepted as input but canonicalized to `no-map` + for Podman's API. + ### Step 6: Check Kubernetes Helm Gateways ```bash diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 646b6320bd..64bb4e4ce7 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -117,7 +117,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| | Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. | -| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, and CDI GPU devices when available. | +| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | @@ -170,7 +170,7 @@ The supervisor must be available inside each sandbox workload: | Runtime | Delivery model | |---|---| | Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | -| Podman | Read-only OCI image volume containing the supervisor binary. | +| Podman | Read-only OCI image volume by default; host-cached bind mount when `userns` is configured. | | Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | | VM | Embedded in the guest rootfs bundle. | | Extension | Defined by the out-of-tree driver. | diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index c12b83cf12..4af864f8fb 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -27,8 +27,8 @@ ipnet = "2" base64 = { workspace = true } chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true } reqwest = { workspace = true, features = ["blocking", "rustls-tls-native-roots"], optional = true } -tar = "0.4" -tempfile = "3" +tar = { version = "0.4", optional = true } +tempfile = { version = "3", optional = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } @@ -38,6 +38,7 @@ default = ["telemetry"] ## Compile in anonymous telemetry emission support. On by default; disable with ## `--no-default-features` (plus any other features you need) for a build that ## contains no telemetry endpoint, no HTTP client, and no emission code at all. +driver-extraction = ["dep:tar", "dep:tempfile"] telemetry = ["dep:reqwest", "dep:chrono"] [build-dependencies] diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 65d9641488..f7c9360a1a 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -424,6 +424,7 @@ pub fn supervisor_image_should_refresh(image: &str) -> bool { // Supervisor binary extraction helpers (shared by Docker and Podman drivers) // --------------------------------------------------------------------------- +#[cfg(feature = "driver-extraction")] /// Extract the payload of the first regular-file entry in a tar archive. /// /// Container archive endpoints return a single-file tar when `path` points to @@ -454,6 +455,7 @@ pub fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { Ok(bytes) } +#[cfg(feature = "driver-extraction")] /// Atomically write `bytes` to `final_path` via a sibling temp file. /// /// Creates parent directories as needed. The temp file is synced, `chmod 755` diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index d09edc3968..92a32c6d20 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true repository.workspace = true [dependencies] -openshell-core = { path = "../openshell-core", default-features = false } +openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } tokio = { workspace = true } tonic = { workspace = true } diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index e46d2eed85..c989c13945 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -15,7 +15,7 @@ name = "openshell-driver-podman" path = "src/main.rs" [dependencies] -openshell-core = { path = "../openshell-core", default-features = false } +openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 965a295d19..788ba5506f 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -360,6 +360,7 @@ Podman resources after out-of-band container removal or label drift. | `OPENSHELL_SANDBOX_PROXY_AUTH_FILE` | `--sandbox-proxy-auth-file` | unset | Path to a file containing the proxy credentials as `user:pass`. Staged as a root-only Podman secret so credentials never appear in config or container metadata. Requires the insecure-auth acknowledgement below. | | `OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE` | `--sandbox-proxy-auth-allow-insecure` | unset | Explicit acknowledgement (`true`) that the credential is sent as cleartext Basic auth over the plain-TCP connection to the `http://` proxy. Required when the auth file is set; rejected when it is not. | | `OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME` | `--sandbox-proxy-connect-by-hostname` | unset | Send the destination hostname in CONNECT requests instead of a validated IP. Last resort for proxies whose ACLs filter on hostnames: the proxy then resolves the name itself, so sandbox SSRF/`allowed_ips` validation no longer binds the connection. | +| `OPENSHELL_PODMAN_USERNS` | `--userns` | unset | User namespace mode for sandbox containers (e.g. `auto`). When unset, containers use the default user namespace. | Through the gateway, the same settings are the `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 9fe39cf7e2..508b604ce7 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -513,6 +513,32 @@ impl PodmanClient { } } + /// Download a file from a container as a tar archive. + /// + /// Calls `GET /libpod/containers/{name}/archive?path={path}` and returns + /// the raw tar bytes. The container does not need to be running. + pub async fn copy_from_container( + &self, + name: &str, + path: &str, + ) -> Result { + validate_name(name)?; + let encoded_path = url_encode(path); + let (status, bytes) = self + .request( + hyper::Method::GET, + &format!("/libpod/containers/{name}/archive?path={encoded_path}"), + None, + API_TIMEOUT, + ) + .await?; + if status.is_success() { + Ok(bytes) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + /// Inspect a container by name or ID. pub async fn inspect_container(&self, name: &str) -> Result { validate_name(name)?; diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 50311836ce..783611e507 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -185,10 +185,57 @@ pub struct PodmanComputeConfig { /// pointing the gateway host at the corporate resolver so validated-IP /// CONNECT works in split-horizon networks. pub proxy_connect_by_hostname: Option, + /// User namespace mode for sandbox containers (e.g. `auto`, `private`). + /// When unset, containers use the default user namespace. + pub userns: Option, + /// Explicit UID mappings for `userns = "private"`. + /// Each entry is `"container_id:host_id:size"`. + #[serde(default)] + pub uidmap: Vec, + /// Explicit GID mappings for `userns = "private"`. + /// Each entry is `"container_id:host_id:size"`. + #[serde(default)] + pub gidmap: Vec, } pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; +/// Parse a single `"container_id:host_id:size"` mapping entry. +/// +/// Returns `(container_id, host_id, size)` on success. +pub fn parse_id_map_entry( + field: &str, + entry: &str, +) -> Result<(u32, u32, u32), crate::client::PodmanApiError> { + let parts: Vec<&str> = entry.split(':').collect(); + if parts.len() != 3 { + return Err(crate::client::PodmanApiError::InvalidInput(format!( + "{field} entry '{entry}' must be 'container_id:host_id:size'", + ))); + } + let container_id: u32 = parts[0].parse().map_err(|_| { + crate::client::PodmanApiError::InvalidInput(format!( + "{field} entry '{entry}': container_id must be a non-negative integer", + )) + })?; + let host_id: u32 = parts[1].parse().map_err(|_| { + crate::client::PodmanApiError::InvalidInput(format!( + "{field} entry '{entry}': host_id must be a non-negative integer", + )) + })?; + let size: u32 = parts[2].parse().map_err(|_| { + crate::client::PodmanApiError::InvalidInput(format!( + "{field} entry '{entry}': size must be a non-negative integer", + )) + })?; + if size == 0 { + return Err(crate::client::PodmanApiError::InvalidInput(format!( + "{field} entry '{entry}': size must be greater than 0", + ))); + } + Ok((container_id, host_id, size)) +} + impl PodmanComputeConfig { /// Returns `true` when all three TLS paths are configured. #[must_use] @@ -328,6 +375,88 @@ impl PodmanComputeConfig { Ok(()) } + /// Validate and canonicalize the optional `userns` mode. + /// + /// Supported modes: `auto` (with optional params, e.g. `auto:size=65536`), + /// `host`, `keep-id` (with optional params), `no-map` (alias `nomap`), + /// and `private` (requires explicit `uidmap`/`gidmap`). + /// Modes that don't accept parameters (`host`, `no-map`, `private`) are + /// rejected when a colon-separated suffix is present. + /// + /// On success, `self.userns` is rewritten with the canonical lowercase + /// mode string so downstream code can rely on exact matches. + pub fn canonicalize_userns(&mut self) -> Result<(), crate::client::PodmanApiError> { + let Some(mode) = self.userns.as_deref() else { + return Ok(()); + }; + let (base, has_params) = mode + .split_once(':') + .map_or((mode, false), |(b, _)| (b, true)); + let canonical = match base.to_ascii_lowercase().as_str() { + "auto" => "auto", + "host" => "host", + "keep-id" => "keep-id", + "nomap" | "no-map" => "no-map", + "private" => "private", + _ => { + return Err(crate::client::PodmanApiError::InvalidInput(format!( + "unsupported userns mode '{mode}'; \ + supported modes: auto, host, keep-id, no-map, private", + ))); + } + }; + if has_params { + match canonical { + "auto" | "keep-id" => {} + _ => { + return Err(crate::client::PodmanApiError::InvalidInput(format!( + "userns mode '{canonical}' does not accept parameters", + ))); + } + } + } + self.userns = Some(if has_params { + let params = mode.split_once(':').unwrap().1; + format!("{canonical}:{params}") + } else { + canonical.to_string() + }); + Ok(()) + } + + /// Validate `uidmap`/`gidmap` consistency with the userns mode. + /// + /// `private` requires at least one entry in both `uidmap` and `gidmap`; + /// other modes (or no userns) reject non-empty mappings. Each entry must + /// be `"container_id:host_id:size"` with `size > 0`. + pub fn validate_userns_mappings(&self) -> Result<(), crate::client::PodmanApiError> { + let is_private = self + .userns + .as_deref() + .is_some_and(|m| m.eq_ignore_ascii_case("private")); + + if is_private { + if self.uidmap.is_empty() || self.gidmap.is_empty() { + return Err(crate::client::PodmanApiError::InvalidInput( + "userns mode 'private' requires at least one entry in both \ + uidmap and gidmap" + .to_string(), + )); + } + } else if !self.uidmap.is_empty() || !self.gidmap.is_empty() { + return Err(crate::client::PodmanApiError::InvalidInput( + "uidmap/gidmap are only valid with userns = \"private\"".to_string(), + )); + } + + for (field, entries) in [("uidmap", &self.uidmap), ("gidmap", &self.gidmap)] { + for entry in entries { + parse_id_map_entry(field, entry)?; + } + } + Ok(()) + } + /// Validate optional host gateway override. pub fn validate_host_gateway_ip(&self) -> Result<(), crate::client::PodmanApiError> { let trimmed = self.host_gateway_ip.trim(); @@ -380,6 +509,9 @@ impl Default for PodmanComputeConfig { proxy_auth_file: None, proxy_auth_allow_insecure: None, proxy_connect_by_hostname: None, + userns: None, + uidmap: Vec::new(), + gidmap: Vec::new(), } } } @@ -412,6 +544,9 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("proxy_auth_file", &self.proxy_auth_file.is_some()) .field("proxy_auth_allow_insecure", &self.proxy_auth_allow_insecure) .field("proxy_connect_by_hostname", &self.proxy_connect_by_hostname) + .field("userns", &self.userns) + .field("uidmap", &self.uidmap) + .field("gidmap", &self.gidmap) .finish() } } @@ -794,4 +929,174 @@ mod tests { assert!(!msg.contains("OPENSHELL_PODMAN_TLS_CERT"), "{msg}"); assert!(!msg.contains("OPENSHELL_PODMAN_TLS_KEY"), "{msg}"); } + + #[test] + fn canonicalize_userns_accepts_supported_modes() { + for mode in [ + "auto", + "host", + "keep-id", + "no-map", + "private", + "auto:size=65536", + "keep-id:uid=1000,gid=1000", + ] { + let mut cfg = PodmanComputeConfig { + userns: Some(mode.to_string()), + ..PodmanComputeConfig::default() + }; + cfg.canonicalize_userns() + .unwrap_or_else(|_| panic!("mode '{mode}' should be accepted")); + } + } + + #[test] + fn canonicalize_userns_normalizes_case_and_aliases() { + let cases = [ + ("Auto", "auto"), + ("HOST", "host"), + ("KEEP-ID:uid=1000", "keep-id:uid=1000"), + ("nomap", "no-map"), + ("no-map", "no-map"), + ("Private", "private"), + ("auto:SIZE=65536", "auto:SIZE=65536"), + ]; + for (input, expected) in cases { + let mut cfg = PodmanComputeConfig { + userns: Some(input.to_string()), + ..PodmanComputeConfig::default() + }; + cfg.canonicalize_userns() + .unwrap_or_else(|_| panic!("mode '{input}' should be accepted")); + assert_eq!( + cfg.userns.as_deref(), + Some(expected), + "input '{input}' should canonicalize to '{expected}'" + ); + } + } + + #[test] + fn canonicalize_userns_rejects_unsupported_modes() { + for mode in ["container:foo", "ns:/proc/1/ns/user", "4000:5000"] { + let mut cfg = PodmanComputeConfig { + userns: Some(mode.to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg + .canonicalize_userns() + .expect_err(&format!("mode '{mode}' should be rejected")); + let msg = err.to_string(); + assert!(msg.contains("unsupported userns mode"), "{msg}"); + } + } + + #[test] + fn canonicalize_userns_rejects_params_on_non_parameterizable_modes() { + for mode in ["host:foo", "no-map:x=1", "private:x=1"] { + let mut cfg = PodmanComputeConfig { + userns: Some(mode.to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg + .canonicalize_userns() + .expect_err(&format!("mode '{mode}' should be rejected")); + let msg = err.to_string(); + assert!(msg.contains("does not accept parameters"), "{msg}"); + } + } + + #[test] + fn canonicalize_userns_accepts_none() { + let mut cfg = PodmanComputeConfig::default(); + cfg.canonicalize_userns().expect("None should be accepted"); + } + + // ── Userns mapping validation ──────────────────────────────────── + + #[test] + fn validate_userns_mappings_accepts_private_with_maps() { + let cfg = PodmanComputeConfig { + userns: Some("private".to_string()), + uidmap: vec!["0:1000:1".to_string(), "1:100000:65536".to_string()], + gidmap: vec!["0:1000:1".to_string(), "1:100000:65536".to_string()], + ..PodmanComputeConfig::default() + }; + cfg.validate_userns_mappings() + .expect("private with mappings should be accepted"); + } + + #[test] + fn validate_userns_mappings_rejects_private_without_uidmap() { + let cfg = PodmanComputeConfig { + userns: Some("private".to_string()), + gidmap: vec!["0:1000:1".to_string()], + ..PodmanComputeConfig::default() + }; + let err = cfg + .validate_userns_mappings() + .expect_err("private without uidmap should be rejected"); + assert!(err.to_string().contains("uidmap"), "{err}"); + } + + #[test] + fn validate_userns_mappings_rejects_private_without_gidmap() { + let cfg = PodmanComputeConfig { + userns: Some("private".to_string()), + uidmap: vec!["0:1000:1".to_string()], + ..PodmanComputeConfig::default() + }; + let err = cfg + .validate_userns_mappings() + .expect_err("private without gidmap should be rejected"); + assert!(err.to_string().contains("gidmap"), "{err}"); + } + + #[test] + fn validate_userns_mappings_rejects_maps_without_private() { + for mode in [Some("auto"), Some("host"), Some("keep-id"), None] { + let cfg = PodmanComputeConfig { + userns: mode.map(ToString::to_string), + uidmap: vec!["0:1000:1".to_string()], + gidmap: vec!["0:1000:1".to_string()], + ..PodmanComputeConfig::default() + }; + let err = cfg.validate_userns_mappings().expect_err(&format!( + "mappings without private should be rejected (mode={mode:?})" + )); + assert!( + err.to_string().contains("only valid with"), + "{mode:?}: {err}" + ); + } + } + + #[test] + fn validate_userns_mappings_rejects_malformed_entries() { + let cases = [ + ("0:1000", "too few fields"), + ("0:1000:1:extra", "too many fields"), + ("abc:1000:1", "non-numeric container_id"), + ("0:abc:1", "non-numeric host_id"), + ("0:1000:abc", "non-numeric size"), + ("0:1000:0", "zero size"), + ]; + for (entry, desc) in cases { + let cfg = PodmanComputeConfig { + userns: Some("private".to_string()), + uidmap: vec![entry.to_string()], + gidmap: vec!["0:1000:1".to_string()], + ..PodmanComputeConfig::default() + }; + cfg.validate_userns_mappings() + .expect_err(&format!("{desc}: '{entry}' should be rejected")); + } + } + + #[test] + fn validate_userns_mappings_accepts_no_userns_no_maps() { + let cfg = PodmanComputeConfig::default(); + cfg.validate_userns_mappings() + .expect("no userns and no maps should be accepted"); + } } diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 005f688a19..fed525002a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -54,6 +54,9 @@ const VOLUME_PREFIX: &str = "openshell-sandbox-"; /// Secret name prefix for per-sandbox gateway JWTs. const TOKEN_SECRET_PREFIX: &str = "openshell-token-"; const PROXY_AUTH_SECRET_PREFIX: &str = "openshell-proxy-auth-"; +const TLS_CA_SECRET_PREFIX: &str = "openshell-tls-ca-"; +const TLS_CERT_SECRET_PREFIX: &str = "openshell-tls-cert-"; +const TLS_KEY_SECRET_PREFIX: &str = "openshell-tls-key-"; /// Container-side mount paths for client TLS materials and the sandbox token. const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; @@ -171,6 +174,16 @@ pub fn proxy_auth_secret_name(sandbox_id: &str) -> String { format!("{PROXY_AUTH_SECRET_PREFIX}{sandbox_id}") } +/// Build per-sandbox Podman secret names for TLS CA, cert, and key. +#[must_use] +pub fn tls_secret_names(sandbox_id: &str) -> [String; 3] { + [ + format!("{TLS_CA_SECRET_PREFIX}{sandbox_id}"), + format!("{TLS_CERT_SECRET_PREFIX}{sandbox_id}"), + format!("{TLS_KEY_SECRET_PREFIX}{sandbox_id}"), + ] +} + /// Truncate a container ID to 12 characters (standard short form). #[must_use] pub fn short_id(id: &str) -> String { @@ -229,6 +242,13 @@ struct ContainerSpec { /// Port mappings from host to container. Using `host_port=0` requests an /// ephemeral port, readable back from the inspect response. portmappings: Vec, + /// User namespace mode override (e.g. `auto`). + #[serde(skip_serializing_if = "Option::is_none")] + userns: Option, + /// UID/GID mapping options. Required for `userns = "auto"` — the Podman + /// API needs `AutoUserNs: true` alongside the namespace mode. + #[serde(skip_serializing_if = "Option::is_none")] + idmappings: Option, } /// A port mapping entry for the libpod `SpecGenerator`. @@ -328,6 +348,49 @@ struct NetNS { nsmode: String, } +#[derive(Serialize)] +struct UserNS { + nsmode: String, + #[serde(skip_serializing_if = "Option::is_none")] + value: Option, +} + +#[derive(Serialize)] +struct IDMap { + container_id: u32, + host_id: u32, + size: u32, +} + +#[derive(Serialize)] +struct IDMappings { + #[serde(rename = "HostUIDMapping")] + host_uid_mapping: bool, + #[serde(rename = "HostGIDMapping")] + host_gid_mapping: bool, + #[serde(rename = "AutoUserNs")] + auto_user_ns: bool, + #[serde(rename = "UIDMap", skip_serializing_if = "Vec::is_empty")] + uid_map: Vec, + #[serde(rename = "GIDMap", skip_serializing_if = "Vec::is_empty")] + gid_map: Vec, +} + +fn parse_id_maps(entries: &[String]) -> Result, ComputeDriverError> { + entries + .iter() + .map(|entry| { + let (cid, hid, size) = crate::config::parse_id_map_entry("idmap", entry) + .map_err(|err| ComputeDriverError::Precondition(err.to_string()))?; + Ok(IDMap { + container_id: cid, + host_id: hid, + size, + }) + }) + .collect() +} + #[derive(Serialize)] struct NetworkAttachment {} @@ -905,9 +968,12 @@ pub fn build_container_spec_with_token_and_gpu_devices( image, image, "", + None, + None, ) } +#[allow(clippy::too_many_arguments)] pub fn build_container_spec_for_image( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -916,6 +982,8 @@ pub fn build_container_spec_for_image( requested_image: &str, image_id: &str, oci_user: &str, + supervisor_bin_path: Option<&Path>, + tls_secret_names: Option<&[String; 3]>, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); @@ -957,11 +1025,15 @@ pub fn build_container_spec_for_image( }]; volumes.extend(user_mounts.volumes); - let mut image_volumes = vec![ImageVolume { - source: config.supervisor_image.clone(), - destination: SUPERVISOR_MOUNT_DIR.into(), - rw: false, - }]; + let mut image_volumes = if supervisor_bin_path.is_some() { + Vec::new() + } else { + vec![ImageVolume { + source: config.supervisor_image.clone(), + destination: SUPERVISOR_MOUNT_DIR.into(), + rw: false, + }] + }; image_volumes.extend(user_mounts.image_volumes); let mut command = vec![ "--workdir".to_string(), @@ -1111,6 +1183,29 @@ pub fn build_container_spec_for_image( mode: 0o400, }); } + if let Some([ca, cert, key]) = tls_secret_names { + secrets.push(SecretMount { + source: ca.clone(), + target: TLS_CA_MOUNT_PATH.into(), + uid: 0, + gid: 0, + mode: 0o400, + }); + secrets.push(SecretMount { + source: cert.clone(), + target: TLS_CERT_MOUNT_PATH.into(), + uid: 0, + gid: 0, + mode: 0o400, + }); + secrets.push(SecretMount { + source: key.clone(), + target: TLS_KEY_MOUNT_PATH.into(), + uid: 0, + gid: 0, + mode: 0o400, + }); + } secrets }, stop_timeout: config.stop_timeout_secs, @@ -1136,20 +1231,20 @@ pub fn build_container_spec_for_image( destination: openshell_core::container_paths::NETNS_MOUNT_ROOT.into(), options: vec!["rw".into(), "nosuid".into(), "nodev".into()], }]; - // Bind-mount client TLS materials into the container when mTLS - // is enabled. The supervisor reads these via OPENSHELL_TLS_CA, - // OPENSHELL_TLS_CERT, and OPENSHELL_TLS_KEY env vars (set in - // build_env above) to establish an mTLS connection back to the - // gateway. - if let (Some(ca), Some(cert), Some(key)) = ( - &config.guest_tls_ca, - &config.guest_tls_cert, - &config.guest_tls_key, - ) { + // Deliver client TLS materials into the container when mTLS is + // enabled. When userns remaps UIDs (auto, no-map), bind-mounted + // host files are unreadable because the container root maps to a + // different host UID. In that case TLS materials are delivered as + // Podman secrets (handled in the `secrets` block above); otherwise + // use bind mounts. + if tls_secret_names.is_none() + && let (Some(ca), Some(cert), Some(key)) = ( + &config.guest_tls_ca, + &config.guest_tls_cert, + &config.guest_tls_key, + ) + { let mut ro = vec!["ro".into(), "rbind".into()]; - // On SELinux-enabled systems (Fedora, RHEL), bind-mounted - // files need the shared relabel option so the container - // process can read them through the SELinux MAC policy. if is_selinux_enabled() { ro.push("z".into()); } @@ -1172,6 +1267,18 @@ pub fn build_container_spec_for_image( options: ro, }); } + if let Some(bin_path) = supervisor_bin_path { + let mut opts = vec!["ro".into(), "rbind".into()]; + if is_selinux_enabled() { + opts.push("z".into()); + } + m.push(Mount { + kind: "bind".into(), + source: bin_path.display().to_string(), + destination: SUPERVISOR_BINARY_PATH.into(), + options: opts, + }); + } m.extend(user_mounts.mounts); m }, @@ -1183,6 +1290,36 @@ pub fn build_container_spec_for_image( container_port: openshell_core::config::DEFAULT_SSH_PORT, protocol: "tcp".into(), }], + userns: config.userns.as_deref().map(|raw| { + let (base, params) = raw + .split_once(':') + .map_or((raw, None), |(b, p)| (b, Some(p))); + UserNS { + nsmode: base.to_string(), + value: params.map(ToString::to_string), + } + }), + idmappings: match config + .userns + .as_deref() + .map(|m| m.split(':').next().unwrap_or(m)) + { + Some("auto") => Some(IDMappings { + host_uid_mapping: false, + host_gid_mapping: false, + auto_user_ns: true, + uid_map: Vec::new(), + gid_map: Vec::new(), + }), + Some("private") => Some(IDMappings { + host_uid_mapping: false, + host_gid_mapping: false, + auto_user_ns: false, + uid_map: parse_id_maps(&config.uidmap)?, + gid_map: parse_id_maps(&config.gidmap)?, + }), + _ => None, + }, }; Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) @@ -1385,6 +1522,8 @@ mod tests { "registry.example/app:latest", "sha256:immutable", "app:staff", + None, + None, ) .unwrap(); @@ -2787,4 +2926,205 @@ mod tests { .count(); assert_eq!(bind_count, 0, "no bind mounts without TLS config"); } + + #[test] + fn container_spec_includes_userns_when_configured() { + let sandbox = test_sandbox("userns-id", "userns-name"); + let mut config = test_config(); + config.userns = Some("auto".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("auto")); + assert!(userns.get("value").is_none(), "bare auto should omit value"); + + let idmappings = &spec["idmappings"]; + assert_eq!( + idmappings["AutoUserNs"].as_bool(), + Some(true), + "idmappings.AutoUserNs must be true for userns=auto" + ); + } + + #[test] + fn container_spec_auto_with_params() { + let sandbox = test_sandbox("userns-auto-params-id", "userns-auto-params-name"); + let mut config = test_config(); + config.userns = Some("auto:size=65536".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("auto")); + assert_eq!(userns["value"].as_str(), Some("size=65536")); + + assert_eq!( + spec["idmappings"]["AutoUserNs"].as_bool(), + Some(true), + "idmappings.AutoUserNs must be true for auto:size=65536" + ); + } + + #[test] + fn container_spec_keep_id_with_params() { + let sandbox = test_sandbox("userns-keepid-id", "userns-keepid-name"); + let mut config = test_config(); + config.userns = Some("keep-id:uid=1000,gid=1000".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("keep-id")); + assert_eq!(userns["value"].as_str(), Some("uid=1000,gid=1000")); + + assert!( + spec.get("idmappings").is_none(), + "idmappings should not be set for keep-id" + ); + } + + #[test] + fn container_spec_nomap_mode() { + let sandbox = test_sandbox("userns-nomap-id", "userns-nomap-name"); + let mut config = test_config(); + config.userns = Some("no-map".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("no-map")); + assert!(userns.get("value").is_none(), "no-map should omit value"); + + assert!( + spec.get("idmappings").is_none(), + "idmappings should not be set for no-map" + ); + } + + #[test] + fn container_spec_private_with_mappings() { + let sandbox = test_sandbox("userns-private-id", "userns-private-name"); + let mut config = test_config(); + config.userns = Some("private".to_string()); + config.uidmap = vec!["0:1000:1".to_string(), "1:100000:65536".to_string()]; + config.gidmap = vec!["0:1000:1".to_string(), "1:100000:65536".to_string()]; + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("private")); + assert!(userns.get("value").is_none(), "private should omit value"); + + let idmappings = &spec["idmappings"]; + assert_eq!( + idmappings["AutoUserNs"].as_bool(), + Some(false), + "AutoUserNs must be false for private mode" + ); + + let uid_map = idmappings["UIDMap"] + .as_array() + .expect("UIDMap should be an array"); + assert_eq!(uid_map.len(), 2); + assert_eq!(uid_map[0]["container_id"].as_u64(), Some(0)); + assert_eq!(uid_map[0]["host_id"].as_u64(), Some(1000)); + assert_eq!(uid_map[0]["size"].as_u64(), Some(1)); + assert_eq!(uid_map[1]["container_id"].as_u64(), Some(1)); + assert_eq!(uid_map[1]["host_id"].as_u64(), Some(100_000)); + assert_eq!(uid_map[1]["size"].as_u64(), Some(65536)); + + let gid_map = idmappings["GIDMap"] + .as_array() + .expect("GIDMap should be an array"); + assert_eq!(gid_map.len(), 2); + assert_eq!(gid_map[0]["container_id"].as_u64(), Some(0)); + assert_eq!(gid_map[0]["host_id"].as_u64(), Some(1000)); + assert_eq!(gid_map[0]["size"].as_u64(), Some(1)); + } + + #[test] + fn container_spec_omits_userns_when_unset() { + let sandbox = test_sandbox("no-userns-id", "no-userns-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + assert!( + spec.get("userns").is_none(), + "userns should not be set when unconfigured" + ); + assert!( + spec.get("idmappings").is_none(), + "idmappings should not be set when userns is unconfigured" + ); + } + + #[test] + fn container_spec_uses_bind_mount_for_supervisor_when_path_provided() { + let sandbox = test_sandbox("bind-sv-id", "bind-sv-name"); + let config = test_config(); + let image = resolve_image(&sandbox, &config); + let spec = build_container_spec_for_image( + &sandbox, + &config, + None, + None, + image, + image, + "", + Some(Path::new("/host/cache/openshell-sandbox")), + None, + ) + .unwrap(); + + let image_volumes = spec["image_volumes"] + .as_array() + .expect("image_volumes should be an array"); + assert!( + !image_volumes + .iter() + .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), + "supervisor image volume should not be present when bind path is provided" + ); + + let mounts = spec["mounts"] + .as_array() + .expect("mounts should be an array"); + let sv_bind = mounts + .iter() + .find(|m| m["destination"].as_str() == Some(SUPERVISOR_BINARY_PATH)); + assert!( + sv_bind.is_some(), + "supervisor bind mount should be present at {SUPERVISOR_BINARY_PATH}" + ); + let sv_bind = sv_bind.unwrap(); + assert_eq!( + sv_bind["source"].as_str(), + Some("/host/cache/openshell-sandbox") + ); + assert_eq!(sv_bind["type"].as_str(), Some("bind")); + } + + #[test] + fn container_spec_uses_image_volume_when_no_bind_path() { + let sandbox = test_sandbox("imgvol-id", "imgvol-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let image_volumes = spec["image_volumes"] + .as_array() + .expect("image_volumes should be an array"); + assert!( + image_volumes + .iter() + .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), + "supervisor image volume should be present by default" + ); + + let mounts = spec["mounts"] + .as_array() + .expect("mounts should be an array"); + assert!( + !mounts.iter().any( + |m| m["destination"].as_str() == Some(SUPERVISOR_BINARY_PATH) + && m["type"].as_str() == Some("bind") + ), + "supervisor bind mount should not be present by default" + ); + } } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 51c689fb29..c9b4515db6 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -11,7 +11,10 @@ use crate::watcher::{ }; use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; -use openshell_core::driver_utils::supervisor_image_should_refresh; +use openshell_core::driver_utils::{ + SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, + temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, +}; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, @@ -179,6 +182,52 @@ async fn cleanup_sandbox_proxy_auth_secret(client: &PodmanClient, secret_name: & } } +async fn create_tls_secrets( + client: &PodmanClient, + config: &PodmanComputeConfig, + names: &[String; 3], +) -> Result<(), ComputeDriverError> { + let paths = [ + config.guest_tls_ca.as_deref(), + config.guest_tls_cert.as_deref(), + config.guest_tls_key.as_deref(), + ]; + let mut created = 0usize; + for (name, path) in names.iter().zip(paths.iter()) { + let Some(p) = path else { continue }; + let result = async { + let data = std::fs::read(p).map_err(|e| { + ComputeDriverError::Message(format!("read TLS file '{}': {e}", p.display())) + })?; + client + .create_secret(name, &data) + .await + .map_err(ComputeDriverError::from) + } + .await; + if let Err(e) = result { + for prev in &names[..created] { + let _ = client.remove_secret(prev).await; + } + return Err(e); + } + created += 1; + } + Ok(()) +} + +async fn cleanup_tls_secrets(client: &PodmanClient, names: &[String; 3]) { + for name in names { + if let Err(err) = client.remove_secret(name).await { + warn!( + secret = %name, + error = %err, + "Failed to remove TLS secret" + ); + } + } +} + fn local_podman_cdi_gpu_inventory_from(dev_root: &Path) -> CdiGpuInventory { let mut device_ids = std::fs::read_dir(dev_root) .ok() @@ -277,6 +326,8 @@ impl PodmanComputeDriver { config.validate_runtime_limits()?; config.validate_host_gateway_ip()?; config.validate_proxy_config()?; + config.canonicalize_userns()?; + config.validate_userns_mappings()?; let client = PodmanClient::new(socket_path); @@ -755,6 +806,37 @@ impl PodmanComputeDriver { return Err(e); } }; + let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) { + match extract_supervisor_bin(&self.client, &self.config).await { + Ok(path) => Some(path), + Err(e) => { + cleanup_created().await; + return Err(e); + } + } + } else { + None + }; + + let tls_secret_names = + if userns_remaps_uids(self.config.userns.as_deref()) && self.config.tls_enabled() { + let names = container::tls_secret_names(&sandbox.id); + if let Err(e) = create_tls_secrets(&self.client, &self.config, &names).await { + cleanup_created().await; + return Err(e); + } + Some(names) + } else { + None + }; + + let cleanup_all = || async { + cleanup_created().await; + if let Some(names) = &tls_secret_names { + cleanup_tls_secrets(&self.client, names).await; + } + }; + let spec = match container::build_container_spec_for_image( sandbox, &self.config, @@ -763,25 +845,23 @@ impl PodmanComputeDriver { image, &inspected_image.id, image_user, + supervisor_bin_path.as_deref(), + tls_secret_names.as_ref(), ) { Ok(spec) => spec, Err(e) => { - cleanup_created().await; + cleanup_all().await; return Err(e); } }; match self.client.create_container(&spec).await { Ok(_) => {} Err(PodmanApiError::Conflict(_)) => { - // Clean up the volume we just created. It is keyed by *this* - // sandbox's ID, not the conflicting container's ID (which - // has the same name but a different ID), so it would be - // orphaned otherwise. - cleanup_created().await; + cleanup_all().await; return Err(ComputeDriverError::AlreadyExists); } Err(e) => { - cleanup_created().await; + cleanup_all().await; return Err(ComputeDriverError::from(e)); } } @@ -797,7 +877,7 @@ impl PodmanComputeDriver { .client .remove_container(&name, self.config.stop_timeout_secs) .await; - cleanup_created().await; + cleanup_all().await; return Err(ComputeDriverError::from(e)); } @@ -858,6 +938,7 @@ impl PodmanComputeDriver { &container::proxy_auth_secret_name(sandbox_id), ) .await; + cleanup_tls_secrets(&self.client, &container::tls_secret_names(sandbox_id)).await; return Ok(false); }; info!(sandbox_id = %sandbox_id, container = %container_id, "Deleting sandbox container"); @@ -891,6 +972,7 @@ impl PodmanComputeDriver { &container::proxy_auth_secret_name(sandbox_id), ) .await; + cleanup_tls_secrets(&self.client, &container::tls_secret_names(sandbox_id)).await; Ok(container_existed) } @@ -1092,6 +1174,131 @@ fn validate_rootless_local_callback_helper( ))) } +// ── Supervisor binary extraction (userns fallback) ───────────────────── + +async fn extract_supervisor_bin( + client: &PodmanClient, + config: &PodmanComputeConfig, +) -> Result { + let mut inspect = client + .inspect_image(&config.supervisor_image) + .await + .map_err(ComputeDriverError::from)?; + + if supervisor_image_should_refresh(&config.supervisor_image) { + info!( + image = %config.supervisor_image, + "Refreshing mutable podman supervisor image" + ); + match client.pull_image(&config.supervisor_image, "always").await { + Ok(()) => { + inspect = client + .inspect_image(&config.supervisor_image) + .await + .map_err(ComputeDriverError::from)?; + } + Err(err) => { + warn!( + image = %config.supervisor_image, + error = %err, + "Failed to refresh mutable podman supervisor image; \ + falling back to local image if present", + ); + } + } + } + + let digest = if inspect.id.is_empty() { + return Err(ComputeDriverError::Precondition(format!( + "supervisor image '{}' has no ID", + config.supervisor_image, + ))); + } else { + &inspect.id + }; + + let cache_path = + openshell_core::driver_utils::supervisor_cache_path("podman-supervisor", digest) + .map_err(ComputeDriverError::Precondition)?; + if cache_path.is_file() { + validate_linux_elf_binary(&cache_path).map_err(ComputeDriverError::Precondition)?; + info!( + cache_path = %cache_path.display(), + "Using cached supervisor binary" + ); + return Ok(cache_path); + } + + info!( + image = %config.supervisor_image, + cache_path = %cache_path.display(), + "Extracting supervisor binary from image" + ); + + let container_name = temp_extract_container_name(); + let spec = serde_json::json!({ + "image": config.supervisor_image, + "name": container_name, + "entrypoint": [SUPERVISOR_IMAGE_BINARY_PATH], + "command": [], + }); + client + .create_container(&spec) + .await + .map_err(ComputeDriverError::from)?; + + let result = extract_binary_from_container(client, &container_name, &cache_path).await; + + if let Err(err) = client.remove_container(&container_name, 0).await { + warn!( + container = container_name, + error = %err, + "Failed to remove supervisor extractor container" + ); + } + + result +} + +async fn extract_binary_from_container( + client: &PodmanClient, + container_name: &str, + cache_path: &Path, +) -> Result { + let tar_bytes = client + .copy_from_container(container_name, SUPERVISOR_IMAGE_BINARY_PATH) + .await + .map_err(ComputeDriverError::from)?; + + let binary_bytes = extract_first_tar_entry(&tar_bytes).map_err(|err| { + ComputeDriverError::Precondition(format!( + "failed to extract supervisor binary from tar: {err}" + )) + })?; + + write_cache_binary_atomic(cache_path, &binary_bytes) + .map_err(ComputeDriverError::Precondition)?; + validate_linux_elf_binary(cache_path).map_err(ComputeDriverError::Precondition)?; + Ok(cache_path.to_path_buf()) +} + +fn userns_needs_extraction(userns: Option<&str>) -> bool { + userns.is_some_and(|mode| { + let base = mode.split(':').next().unwrap_or(mode); + !base.eq_ignore_ascii_case("host") + }) +} + +/// Returns `true` when userns remaps all UIDs, making host-owned bind mounts +/// unreadable from inside the container. `auto` and `no-map` remap every UID; +/// `keep-id` preserves the host user's UID; `host` uses the host namespace. +fn userns_remaps_uids(userns: Option<&str>) -> bool { + userns.is_some_and(|mode| { + let base = mode.split(':').next().unwrap_or(mode); + !matches!(base.to_ascii_lowercase().as_str(), "host" | "keep-id") + }) +} + #[cfg(test)] mod tests { use super::*; @@ -2245,4 +2452,28 @@ mod tests { ); let _ = fs::remove_file(socket_path); } + + #[test] + fn userns_needs_extraction_cases() { + assert!(!userns_needs_extraction(None)); + assert!(!userns_needs_extraction(Some("host"))); + assert!(!userns_needs_extraction(Some("Host"))); + assert!(userns_needs_extraction(Some("auto"))); + assert!(userns_needs_extraction(Some("auto:size=65536"))); + assert!(userns_needs_extraction(Some("keep-id"))); + assert!(userns_needs_extraction(Some("keep-id:uid=1000"))); + assert!(userns_needs_extraction(Some("no-map"))); + assert!(userns_needs_extraction(Some("private"))); + } + + #[test] + fn userns_remaps_uids_cases() { + assert!(!userns_remaps_uids(None)); + assert!(!userns_remaps_uids(Some("host"))); + assert!(!userns_remaps_uids(Some("keep-id"))); + assert!(userns_remaps_uids(Some("auto"))); + assert!(userns_remaps_uids(Some("auto:size=65536"))); + assert!(userns_remaps_uids(Some("no-map"))); + assert!(userns_remaps_uids(Some("private"))); + } } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index e287075886..405deb93a6 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -133,6 +133,21 @@ struct Args { /// SSRF/`allowed_ips` validation no longer binds the connection. #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] sandbox_proxy_connect_by_hostname: Option, + + /// User namespace mode for sandbox containers (e.g. `auto`). + /// When unset, containers use the default user namespace. + #[arg(long, env = "OPENSHELL_PODMAN_USERNS")] + userns: Option, + + /// Explicit UID mappings for `userns = "private"`. + /// Each entry is `"container_id:host_id:size"`. + #[arg(long = "uidmap")] + uidmap: Vec, + + /// Explicit GID mappings for `userns = "private"`. + /// Each entry is `"container_id:host_id:size"`. + #[arg(long = "gidmap")] + gidmap: Vec, } #[tokio::main] @@ -168,6 +183,9 @@ async fn main() -> Result<()> { proxy_auth_file: args.sandbox_proxy_auth_file, proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, + userns: args.userns, + uidmap: args.uidmap, + gidmap: args.gidmap, ..PodmanComputeConfig::default() }) .await diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index f56d233f2f..f59cbc5858 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -203,6 +203,9 @@ fn apply_podman_env_overrides(podman: &mut PodmanComputeConfig) { if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { podman.host_gateway_ip = ip; } + if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { + podman.userns = Some(mode); + } } fn apply_remote_driver_overrides( diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 4db3c9c472..c695dfddfc 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -457,6 +457,15 @@ sandbox_pids_limit = 2048 # but increase process churn (each check spawns a conmon subprocess). # Set to 0 to disable health checks entirely. Default: 10. health_check_interval_secs = 10 +# User namespace mode for sandbox containers. Omit to use the default. +# Supported modes: auto, host, keep-id, no-map, private. +# userns = "auto" +# Explicit UID/GID mappings for userns = "private". Each entry is +# "container_id:host_id:size". Required when mode is "private"; rejected +# for other modes. Rootless Podman uses intermediate IDs (0:0:1, 1:1:65535); +# rootful Podman uses absolute host IDs (0:1000:1, 1:100000:65536). +# uidmap = ["0:0:1", "1:1:65535"] +# gidmap = ["0:0:1", "1:1:65535"] # Corporate forward proxy for sandbox egress. When set, the in-container # supervisor chains policy-approved TLS tunnels through this proxy with HTTP # CONNECT instead of dialing destinations directly. Plain-HTTP requests are diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 5a8028779a..679e2c326d 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -90,6 +90,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -577,6 +583,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -595,6 +613,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "nix", "prost", "rand", "serde", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index c8ef57f693..1ab410789c 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -78,6 +78,11 @@ name = "podman_oci_identity" path = "tests/podman_oci_identity.rs" required-features = ["e2e-podman"] +[[test]] +name = "podman_userns" +path = "tests/podman_userns.rs" +required-features = ["e2e-podman"] + [[test]] name = "vm_gateway_resume" path = "tests/vm_gateway_resume.rs" @@ -152,6 +157,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = "0.0.12" url = "2" +nix = { version = "0.29", features = ["user"] } [dev-dependencies] serial_test = "3" diff --git a/e2e/rust/tests/podman_userns.rs b/e2e/rust/tests/podman_userns.rs new file mode 100644 index 0000000000..0e01aee818 --- /dev/null +++ b/e2e/rust/tests/podman_userns.rs @@ -0,0 +1,302 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-podman")] + +use std::path::PathBuf; +use std::time::Duration; + +use openshell_e2e::harness::cli::wait_for_healthy; +use openshell_e2e::harness::container::is_e2e_driver; +use openshell_e2e::harness::gateway::ManagedGateway; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serial_test::serial; + +const READY_MARKER: &str = "podman-userns-ready"; + +struct GatewayUsernsConfig { + config_path: PathBuf, + original: Vec, + restored: bool, +} + +impl GatewayUsernsConfig { + fn config_path_from_args() -> Result { + let args_file = std::env::var("OPENSHELL_E2E_GATEWAY_ARGS_FILE") + .map_err(|_| "OPENSHELL_E2E_GATEWAY_ARGS_FILE must be set".to_string())?; + let raw = std::fs::read(&args_file) + .map_err(|err| format!("read gateway args file '{args_file}': {err}"))?; + let args: Vec = raw + .split(|byte| *byte == 0) + .filter(|arg| !arg.is_empty()) + .map(|arg| String::from_utf8_lossy(arg).into_owned()) + .collect(); + args.iter() + .position(|arg| arg == "--config") + .and_then(|index| args.get(index + 1)) + .map(PathBuf::from) + .ok_or_else(|| format!("no --config argument in gateway args file '{args_file}'")) + } + + async fn apply(extra_toml: &str) -> Result { + let config_path = Self::config_path_from_args()?; + let original = std::fs::read(&config_path) + .map_err(|err| format!("read gateway config '{}': {err}", config_path.display()))?; + + let config_str = String::from_utf8_lossy(&original); + let lines: Vec<&str> = config_str.lines().collect(); + + let section_idx = lines + .iter() + .position(|l| { + let t = l.trim(); + t.starts_with('[') && t.contains("openshell.drivers.podman") + }) + .ok_or_else(|| { + format!( + "gateway config '{}' has no [openshell.drivers.podman] section", + config_path.display(), + ) + })?; + + let insert_at = lines[section_idx + 1..] + .iter() + .position(|l| l.trim_start().starts_with('[')) + .map_or(lines.len(), |rel| section_idx + 1 + rel); + + let mut updated = String::new(); + for line in &lines[..insert_at] { + updated.push_str(line); + updated.push('\n'); + } + updated.push_str(extra_toml); + if !extra_toml.ends_with('\n') { + updated.push('\n'); + } + for line in &lines[insert_at..] { + updated.push_str(line); + updated.push('\n'); + } + let updated = updated.into_bytes(); + std::fs::write(&config_path, &updated) + .map_err(|err| format!("write gateway config '{}': {err}", config_path.display()))?; + + let guard = Self { + config_path, + original, + restored: false, + }; + restart_gateway().await?; + Ok(guard) + } + + async fn restore(&mut self) -> Result<(), String> { + if self.restored { + return Ok(()); + } + std::fs::write(&self.config_path, &self.original).map_err(|err| { + format!( + "restore gateway config '{}': {err}", + self.config_path.display() + ) + })?; + restart_gateway().await?; + self.restored = true; + Ok(()) + } +} + +impl Drop for GatewayUsernsConfig { + fn drop(&mut self) { + if self.restored { + return; + } + let _ = std::fs::write(&self.config_path, &self.original); + if let Ok(Some(gateway)) = ManagedGateway::from_env() { + let _ = gateway.stop(); + let _ = gateway.start(); + } + } +} + +fn has_subordinate_ids() -> bool { + if nix::unistd::getuid().is_root() { + return true; + } + let username = std::env::var("USER").unwrap_or_default(); + let uid = nix::unistd::getuid().to_string(); + let Ok(subuid) = std::fs::read_to_string("/etc/subuid") else { + return false; + }; + subuid + .lines() + .any(|l| l.starts_with(&format!("{username}:")) || l.starts_with(&format!("{uid}:"))) +} + +async fn restart_gateway() -> Result<(), String> { + let gateway = ManagedGateway::from_env()? + .ok_or_else(|| "managed gateway metadata disappeared".to_string())?; + gateway.stop()?; + gateway.start()?; + wait_for_healthy(Duration::from_secs(120)).await +} + +#[tokio::test] +#[serial] +async fn podman_userns_keep_id() { + if !is_e2e_driver("podman") { + eprintln!("Skipping Podman userns test: e2e driver is not podman"); + return; + } + if !has_subordinate_ids() { + eprintln!("Skipping: no subordinate UID/GID ranges available"); + return; + } + + let mut userns_config = GatewayUsernsConfig::apply("userns = \"keep-id\"") + .await + .expect("apply userns gateway config"); + + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--no-tty"], + &["sh", "-c", &format!("echo {READY_MARKER}; sleep infinity")], + READY_MARKER, + ) + .await + .expect("create sandbox with userns=keep-id"); + + let id_output = sandbox + .exec(&["id", "-u"]) + .await + .expect("exec id -u in sandbox"); + let sandbox_uid = strip_ansi(&id_output).trim().to_string(); + assert!( + sandbox_uid.parse::().is_ok(), + "sandbox should report a numeric UID, got '{sandbox_uid}'" + ); + + let cat_output = sandbox + .exec(&["cat", "/proc/self/uid_map"]) + .await + .expect("exec cat /proc/self/uid_map in sandbox"); + let uid_map = strip_ansi(&cat_output).trim().to_string(); + let mapping_count = uid_map.lines().count(); + assert!( + mapping_count >= 1, + "userns=keep-id should produce UID mappings, got {mapping_count}: {uid_map}" + ); + + sandbox.cleanup().await; + userns_config + .restore() + .await + .expect("restore gateway config"); +} + +#[tokio::test] +#[serial] +async fn podman_userns_auto() { + if !is_e2e_driver("podman") { + eprintln!("Skipping Podman userns test: e2e driver is not podman"); + return; + } + if !has_subordinate_ids() { + eprintln!("Skipping: no subordinate UID/GID ranges available"); + return; + } + + let mut userns_config = GatewayUsernsConfig::apply("userns = \"auto\"") + .await + .expect("apply userns gateway config"); + + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--no-tty"], + &["sh", "-c", &format!("echo {READY_MARKER}; sleep infinity")], + READY_MARKER, + ) + .await + .expect("create sandbox with userns=auto"); + + let cat_output = sandbox + .exec(&["cat", "/proc/self/uid_map"]) + .await + .expect("exec cat /proc/self/uid_map in sandbox"); + let uid_map = strip_ansi(&cat_output).trim().to_string(); + let mapping_count = uid_map.lines().count(); + assert!( + mapping_count >= 1, + "userns=auto should produce UID mappings, got {mapping_count}: {uid_map}" + ); + + let id_output = sandbox + .exec(&["id", "-u"]) + .await + .expect("exec id -u in sandbox"); + let sandbox_uid = strip_ansi(&id_output).trim().to_string(); + assert!( + sandbox_uid.parse::().is_ok(), + "sandbox should report a numeric UID, got '{sandbox_uid}'" + ); + + sandbox.cleanup().await; + userns_config + .restore() + .await + .expect("restore gateway config"); +} + +#[tokio::test] +#[serial] +async fn podman_userns_private() { + if !is_e2e_driver("podman") { + eprintln!("Skipping Podman userns test: e2e driver is not podman"); + return; + } + if !has_subordinate_ids() { + eprintln!("Skipping: no subordinate UID/GID ranges available"); + return; + } + + let extra_config = "userns = \"private\"\n\ + uidmap = [\"0:0:1\", \"1:1:65535\"]\n\ + gidmap = [\"0:0:1\", \"1:1:65535\"]"; + let mut userns_config = GatewayUsernsConfig::apply(&extra_config) + .await + .expect("apply userns gateway config"); + + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--no-tty"], + &["sh", "-c", &format!("echo {READY_MARKER}; sleep infinity")], + READY_MARKER, + ) + .await + .expect("create sandbox with userns=private"); + + let id_output = sandbox + .exec(&["id", "-u"]) + .await + .expect("exec id -u in sandbox"); + let sandbox_uid = strip_ansi(&id_output).trim().to_string(); + assert!( + sandbox_uid.parse::().is_ok(), + "sandbox should report a numeric UID, got '{sandbox_uid}'" + ); + + let cat_output = sandbox + .exec(&["cat", "/proc/self/uid_map"]) + .await + .expect("exec cat /proc/self/uid_map in sandbox"); + let uid_map = strip_ansi(&cat_output).trim().to_string(); + let mapping_count = uid_map.lines().count(); + assert!( + mapping_count >= 2, + "userns=private should produce at least 2 UID mappings, got {mapping_count}: {uid_map}" + ); + + sandbox.cleanup().await; + userns_config + .restore() + .await + .expect("restore gateway config"); +}