diff --git a/.gitignore b/.gitignore index 04b6cc1..1c7af81 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,12 @@ bin/* Dockerfile.cross +# Locally built command binaries. `go build ./cmd/...` drops these in the repo +# ROOT, not bin/, so they are not covered above — and they are multi-MB, which is +# exactly the kind of thing that sneaks into a commit unnoticed. +/manager +/keybroker + # Test binary, build with `go test -c` *.test diff --git a/.golangci.yml b/.golangci.yml index d25815d..d582d45 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,6 +32,12 @@ linters: - linters: - lll path: api/* + # kubebuilder markers are single-line BY PROTOCOL — controller-gen parses one + # marker per comment line and there is no continuation syntax, so a long + # groups/resources/verbs triple simply cannot be wrapped to satisfy lll. + - linters: + - lll + source: '^\s*//\s*\+kubebuilder:' - linters: - dupl - lll diff --git a/Makefile b/Makefile index a648480..a31997c 100644 --- a/Makefile +++ b/Makefile @@ -95,8 +95,8 @@ test: manifests generate fmt vet setup-envtest ## Run tests. # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# CertManager is installed by default; skip with: -# - CERT_MANAGER_INSTALL_SKIP=true +# No cert-manager is needed: the manager provisions its own webhook serving cert +# in-process (pkg/cert), so the suite only needs a Kind cluster. KIND_CLUSTER ?= nebula-test-e2e .PHONY: setup-test-e2e diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index d27ae8b..bd4dbb4 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -8,6 +8,15 @@ // NodeClaim - one provisioned external instance and its lifecycle. Owns the // terminate finalizer so a paid instance is never leaked. // +// On top of that provisioning core sit the workload types, each synthesizing +// Pods onto the same placement path rather than bypassing it: +// +// Sandbox - one interactive remote box (agent workspace, shell, scratch GPU), +// reachable with the same kubectl exec/logs as a local Pod. +// SandboxSet - maintains N Sandboxes, and owns /scale so `kubectl scale` and HPA +// drive the count. Keeping boxes ready ahead of demand is a USE of +// this, not its definition — there are no lease semantics here. +// // +kubebuilder:object:generate=true // +groupName=nebula.inftyai.com package v1alpha1 @@ -35,6 +44,13 @@ const ( // objectSelector so only opted-in Pods ever hit the mutating webhook. EnabledLabel = "nebula.inftyai.com/enabled" + // EnabledValue is the only value of EnabledLabel that opts a Pod in. The + // comparison is exact, so a Pod labelled "True" or "1" is NOT opted in — the + // label is the webhook's objectSelector, and the API server matches it + // literally, so anything else would make the controllers and the selector + // disagree about which Pods are Nebula's. + EnabledValue = "true" + // ProviderSelectionGate is the scheduling gate the webhook injects at Pod // CREATE. The placement controller removes it once it has chosen a // provider (by adding a provider nodeSelector), releasing the Pod to the @@ -59,6 +75,18 @@ const ( // value is the NodePool name, so the key mirrors the CRD kind. PoolLabel = "nebula.inftyai.com/nodepool" + // SandboxLabel records which Sandbox a Pod belongs to. Its value is the Sandbox + // name, so the key mirrors the CRD kind. The Sandbox controller selects its own + // Pod by it, and it is what makes `kubectl get pods -l + // nebula.inftyai.com/sandbox=alice` work. + SandboxLabel = "nebula.inftyai.com/sandbox" + + // SandboxSetLabel records which SandboxSet created a Sandbox. Its value is the + // set name. It is the selector the set's /scale subresource publishes in status + // (so HPA can find the set's members) and how the set controller enumerates the + // boxes it owns — ownerReferences alone would not support a label-selector query. + SandboxSetLabel = "nebula.inftyai.com/sandboxset" + // AcceleratorTypeLabel carries the requested accelerator TYPE only (e.g. // "a100-40gb" or "h100"). The COUNT is expressed separately as a standard // resource request/limit on the container (nvidia.com/gpu for the NVIDIA @@ -111,6 +139,18 @@ const ( // flows the other way — VK writes it for operators/tooling to read. EndpointAnnotation = "nebula.inftyai.com/endpoint" + // SanddPath is where the SandD binary is found INSIDE a Nebula-provisioned + // container, and therefore the command every synthesized workload Pod runs. It + // is a shared constant rather than a per-adapter string because it is a contract + // with two ends that must agree exactly: the controller writes it as the Pod's + // container command, and every provider bootstrap must make the binary appear at + // this path (the AWS adapter bind-mounts it from the host into the container). + // + // The path lives under /nebula rather than /usr/local/bin to avoid colliding with + // anything the user's own image ships, since the image is arbitrary and we are + // injecting into it. + SanddPath = "/nebula/sandd" + // TerminateInstanceFinalizer is held by every NodeClaim to guarantee teardown. // The virtual kubelet owns the happy path (DeletePod → provider.Terminate, // keyed on the Pod-derived claim name), but its teardown is edge-triggered and diff --git a/api/v1alpha1/sandbox_types.go b/api/v1alpha1/sandbox_types.go new file mode 100644 index 0000000..09ec832 --- /dev/null +++ b/api/v1alpha1/sandbox_types.go @@ -0,0 +1,267 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SandboxSpec is one long-lived, interactive remote box: an agent's workspace, a +// shell, a scratch GPU machine. It is the first workload class Nebula serves +// beyond a hand-written Pod. +// +// A Sandbox is SINGULAR — one object, one instance. There is deliberately no +// replicas field and no pod template, because a sandbox is not fungible: someone +// is attached to it, it accumulates state in its filesystem, and the object's own +// name IS its stable identity (`kubectl exec sandbox-alice` always reaches the +// same box). Set-shaped controllers exist because containers are interchangeable, +// which is exactly the property a sandbox lacks — a rolling update would evict a +// live session, and "scale in by one" would have to guess whose box to kill. A +// caller that wants N boxes creates N Sandboxes, each with its own image, +// lifetime and identity. +// +// The count lives one level up, in SandboxSet, which maintains N Sandboxes and +// owns /scale so `kubectl scale` and HPA work. That split is what lets this type +// stay singular: because the set creates Sandbox OBJECTS rather than replicas +// inside one object, everything that depends on a box being its own object — +// per-box RBAC (grant a user their sandbox and not their neighbour's), a +// per-box image and TTL, and a failure that stays visible instead of being +// papered over by a replacement — keeps working underneath a pool. +// +// The spec deliberately reuses corev1 types (ResourceRequirements, EnvVar) rather +// than inventing parallel fields. The controller synthesizes a Pod, so anything +// it accepts must ultimately BE PodSpec-shaped; re-declaring resources or env +// would fork the vocabulary and, worse, fork the source of truth for the +// accelerator COUNT — which placement and the scheduler's fit check both read +// from the container's nvidia.com/gpu limit (see util.AcceleratorRequest). +// +// The CEL rule below rejects a GPU count with no accelerator type. That pair is +// contradictory rather than merely incomplete — util.AcceleratorRequest returns an +// error for it — so without the rule the object is admitted and then fails at +// PLACEMENT, minutes later and one object removed from the mistake. Note the +// inverse is fine and deliberately allowed: a type with no count means one +// accelerator. +// +kubebuilder:validation:XValidation:rule="has(self.acceleratorType) || !has(self.resources) || ((!has(self.resources.limits) || !('nvidia.com/gpu' in self.resources.limits)) && (!has(self.resources.requests) || !('nvidia.com/gpu' in self.resources.requests)))",message="nvidia.com/gpu requires acceleratorType to be set" +type SandboxSpec struct { + // NodePoolRef names the NodePool whose policy places this sandbox: which + // providers are allowed, which capacity tiers, how to rank them. Required — + // there is no implicit default pool, because placing a paid GPU instance + // against a guessed policy is not a safe default. + // +kubebuilder:validation:MinLength=1 + NodePoolRef string `json:"nodePoolRef"` + + // Image is the container image the sandbox runs. It defaults to a plain Ubuntu, + // because unlike the accelerator the image is not a decision a caller has to make + // to get a useful box: `kubectl exec` into a bare distro is exactly the "give me a + // remote shell" case, and anything else can be installed from inside it. Defaulting + // a paid GPU shape would be guessing at spend; defaulting a shell is not. + // + // Note it deliberately does NOT default to a CUDA image even when an accelerator is + // requested. A conditional default would make the image depend on another field, + // which structural-schema defaulting cannot express and which would surprise anyone + // reading the object back. Ask for a CUDA image explicitly when you want one. + // + // There is deliberately no command field, and one cannot be set: the CRD is a + // structural schema, so `command:` in a Sandbox spec is rejected as an unknown + // field by the apiserver itself — no webhook required. That is not a + // simplification, it is the process model: the container's command is always + // SandD, which runs as PID 1 and is what makes `kubectl exec` and `kubectl logs` + // work against an instance in another cloud. A user-supplied command would + // displace it and take both with it. + // + // A sandbox has nothing to run at boot anyway — the whole point is that commands + // arrive later, over exec — so SandD spawns no child here; it holds the container + // open and serves requests. Workload classes that do run something get it spawned + // as SandD's child instead, which is how it comes to own their stdout/stderr. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:default="ubuntu:24.04" + // +optional + Image string `json:"image,omitempty"` + + // AcceleratorType is the requested accelerator TYPE (e.g. "a100-40gb", + // "h100"), matched case-insensitively against the provider catalog. The COUNT + // is NOT here: it is a standard nvidia.com/gpu entry in Resources, so exactly + // one number drives scheduling fit and provisioning. The controller stamps + // this onto the synthesized Pod's AcceleratorTypeLabel, so a Sandbox and a + // hand-written Nebula Pod go through identical placement. + // + // Empty means a CPU-only sandbox, which is a legitimate (and cheap) thing to + // want for a shell or an agent that only needs a filesystem. + // +optional + AcceleratorType string `json:"acceleratorType,omitempty"` + + // Resources is the standard Kubernetes resource requirements for the sandbox + // container, verbatim. The accelerator count rides here as an nvidia.com/gpu + // limit (`limits: {nvidia.com/gpu: "1"}`), which is where both the placement + // controller and the scheduler already read it from. + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // Env is passed to the sandbox container verbatim, including valueFrom + // references — a sandbox usually needs at least a registry or Hugging Face + // token, and re-inventing secret indirection here would be strictly worse than + // reusing the field everyone already knows. + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` + + // TTL bounds the sandbox's total lifetime, measured from the moment it first + // became Ready (NOT from creation, so a slow provision does not eat into the + // user's time). On expiry the controller releases the instance and the sandbox + // reports phase Expired. + // + // This exists because the failure mode of a remote GPU box is financial: an + // abandoned sandbox bills until someone notices. Omit it for an unbounded + // sandbox, which is a deliberate choice rather than the default. + // +optional + TTL *metav1.Duration `json:"ttl,omitempty"` +} + +// SandboxPhase is the coarse, user-facing lifecycle state, derived from the +// synthesized Pod rather than tracked independently. The Pod (via the virtual +// kubelet) is the source of truth for what the external instance is doing — see +// pkg/vnode/status.go — so this is a projection, and the vocabulary intentionally +// mirrors the Pod status reasons the vnode stamps. +type SandboxPhase string + +const ( + // SandboxPending: the sandbox exists but its Pod has not been placed yet — + // typically waiting on the provider-selection gate, e.g. because no provider in + // the pool can currently serve the requested accelerator. A sandbox that sits + // here points at placement, not at the provider. + SandboxPending SandboxPhase = "Pending" + // SandboxProvisioning: a provider Provision call is in flight; the external + // instance does not exist yet. + SandboxProvisioning SandboxPhase = "Provisioning" + // SandboxInitializing: the instance exists at the provider but is not yet + // reachable — booting, or up but not yet passing reachability checks. Kept + // distinct from Provisioning so a stuck sandbox distinguishes "cannot get + // capacity" from "capacity granted, slow boot". + SandboxInitializing SandboxPhase = "Initializing" + // SandboxReady: the instance is running and reachable. This is the only phase + // in which exec/logs can succeed. + SandboxReady SandboxPhase = "Ready" + // SandboxFailed: the instance failed or vanished (terminated out-of-band, + // reclaimed, or the provision was rejected). Terminal: a sandbox holds + // filesystem state that a fresh instance would not have, so it is never + // silently recreated underneath its user. Delete and recreate it explicitly. + SandboxFailed SandboxPhase = "Failed" + // SandboxExpired: spec.TTL elapsed and the instance was released. Terminal, and + // deliberately not garbage: the object stays as the record of why the box went + // away, so a user who returns to a dead sandbox gets an answer instead of a + // NotFound. + SandboxExpired SandboxPhase = "Expired" +) + +// Sandbox condition types (standard Kubernetes condition convention). +const ( + // SandboxConditionReady is True exactly when the sandbox is usable — the + // instance is running and reachable. It is the condition to wait on + // (`kubectl wait --for=condition=Ready sandbox/x`) and mirrors the Pod's own + // Ready condition. + SandboxConditionReady = "Ready" +) + +// Sandbox condition reasons. +const ( + // ReasonSandboxReady: the instance is running and reachable. + ReasonSandboxReady = "Ready" + // ReasonSandboxProvisioning: still bringing the instance up (covers both + // placement and boot; the phase distinguishes them). + ReasonSandboxProvisioning = "Provisioning" + // ReasonSandboxFailed: the instance failed, was rejected, or vanished. + ReasonSandboxFailed = "Failed" + // ReasonSandboxExpired: spec.TTL elapsed and the instance was released. + ReasonSandboxExpired = "Expired" + // ReasonPodConflict: a Pod of the required name already exists and is NOT owned + // by this Sandbox. The controller refuses to adopt it — it could be an unrelated + // workload, and adopting would hand someone else's Pod a terminate finalizer — + // so the sandbox surfaces the collision instead of acting on a guess. + ReasonPodConflict = "PodConflict" +) + +// SandboxStatus is the observed state, projected from the synthesized Pod. +type SandboxStatus struct { + // Phase is the coarse lifecycle state. + // +optional + Phase SandboxPhase `json:"phase,omitempty"` + + // PodName is the synthesized Pod backing this sandbox. It is recorded even + // though it currently equals the Sandbox name, so tooling (and `kubectl exec` + // wrappers) read the pod identity from status rather than reconstructing it + // from a naming convention this controller would then be unable to change. + // +optional + PodName string `json:"podName,omitempty"` + + // Endpoint is the reachable address of the external instance once it is + // running, in the provider's own form (a public DNS name or an IP). Mirrored + // from the Pod's EndpointAnnotation, which is where the virtual kubelet + // publishes it. + // +optional + Endpoint string `json:"endpoint,omitempty"` + + // ReadyTime is when the sandbox first became Ready. It is the anchor TTL is + // measured from, so it is durable status rather than a derived value: if it + // were recomputed from the Pod, a Pod status blip could silently restart the + // user's clock. + // +optional + ReadyTime *metav1.Time `json:"readyTime,omitempty"` + + // ExpiryTime is when TTL will elapse (ReadyTime + TTL), surfaced so a user can + // see the deadline without doing the arithmetic. Absent when no TTL is set or + // the sandbox has not become Ready yet. + // +optional + ExpiryTime *metav1.Time `json:"expiryTime,omitempty"` + + // Conditions follows the standard Kubernetes condition convention. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=sbx +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Pool",type=string,JSONPath=`.spec.nodePoolRef` +// +kubebuilder:printcolumn:name="Accelerator",type=string,JSONPath=`.spec.acceleratorType` +// +kubebuilder:printcolumn:name="Endpoint",type=string,JSONPath=`.status.endpoint` +// +kubebuilder:printcolumn:name="Expires",type=date,JSONPath=`.status.expiryTime` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// Sandbox is one interactive remote instance — an agent workspace, a shell, a +// scratch GPU box — reachable with the same `kubectl exec` / `kubectl logs` a +// local Pod would be. +type Sandbox struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SandboxSpec `json:"spec,omitempty"` + Status SandboxStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// SandboxList contains a list of Sandbox. +type SandboxList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Sandbox `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Sandbox{}, &SandboxList{}) +} diff --git a/api/v1alpha1/sandboxset_types.go b/api/v1alpha1/sandboxset_types.go new file mode 100644 index 0000000..648dfa3 --- /dev/null +++ b/api/v1alpha1/sandboxset_types.go @@ -0,0 +1,173 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SandboxSetSpec maintains N Sandboxes. That is the whole contract, and the name +// says exactly that much: it is a SET, not a pool. A pool would imply lease +// semantics — claim a box, hold it, return it, with the pool tracking who has what +// — and none of that is implemented here. Keeping N boxes alive is what ENABLES +// warm pooling (holding instances ready because provisioning takes minutes while +// an agent's exec call wants sub-second) and fan-out ("twenty boxes for this +// batch"), but those are uses of a set, not the set's job. "Pool" would also be a +// third meaning of that word in this API group, where NodePool already means +// placement policy. +// +// It creates Sandbox OBJECTS, not replicas inside itself, and that is the point of +// having two types. The count is a genuinely different concern from the box: a set +// answers "how many", a Sandbox answers "which one, running what, for whom". +// Because each box stays its own object underneath, per-box RBAC, per-box status +// and a failure that stays visible all keep working — none of which survives being +// flattened into a replica index. +// +// Boxes get GENERATED names (myset-a4f2x), not ordinals. Ordinals would imply a +// slot that gets refilled, so a box that died would be replaced by an empty one +// wearing the same name — the same address with a different filesystem, which is +// the most confusing thing this API could do. A generated name means a replacement +// is visibly a NEW box, and callers that need a stable handle hold the Sandbox name +// they were given rather than an index into a set. +type SandboxSetSpec struct { + // Replicas is how many Sandboxes to maintain. Zero is legal and useful: it + // releases every box while keeping the set's definition, which is how a set is + // parked overnight without being forgotten. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=1 + Replicas int32 `json:"replicas,omitempty"` + + // Template is the shape of every Sandbox this set creates. All boxes in one set + // are the same shape by construction — the set exists to make boxes + // interchangeable at the point of HANDOUT, so a caller can take any ready box + // without inspecting it. Two shapes means two sets. + // + // A template is right here for the same reason it is wrong on Sandbox itself: + // this object does not describe a box, it describes how to make them. + Template SandboxTemplateSpec `json:"template"` +} + +// SandboxTemplateSpec is the Sandbox a set stamps out: the standard Kubernetes +// template shape (metadata + spec), so created boxes can carry the labels a caller +// selects them by. +type SandboxTemplateSpec struct { + // Metadata is the labels and annotations applied to each created Sandbox. Only + // labels and annotations are honoured; a name here is ignored, since names are + // generated per box. + // +optional + Metadata SandboxTemplateMetadata `json:"metadata,omitempty"` + + // Spec is the SandboxSpec of every box in the set. + Spec SandboxSpec `json:"spec"` +} + +// SandboxTemplateMetadata is the subset of ObjectMeta a template may set. It is +// spelled out rather than embedding metav1.ObjectMeta because embedding would +// advertise fields a template cannot honour (name, ownerReferences, +// resourceVersion) and bloat the CRD schema with them. +type SandboxTemplateMetadata struct { + // Labels are applied to each created Sandbox, on top of the set-ownership + // labels the controller adds. + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations are applied to each created Sandbox. + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// SandboxSet condition types (standard Kubernetes condition convention). +const ( + // SandboxSetConditionReady is True when every desired box is Ready. Callers that + // can start work with a partially ready set should read status.ReadyReplicas + // instead of waiting on this. + SandboxSetConditionReady = "Ready" +) + +// SandboxSet condition reasons. +const ( + // ReasonSandboxSetReady: every desired box is Ready. + ReasonSandboxSetReady = "Ready" + // ReasonSandboxSetProgressing: at least one box is still coming up. Not an error + // — a cold set takes minutes by nature, since each box is a real instance. + ReasonSandboxSetProgressing = "Progressing" + // ReasonSandboxSetScaledToZero: spec.Replicas is 0, so there is nothing to be + // ready. Distinguished from Progressing so a parked set does not read as a stuck + // one. + ReasonSandboxSetScaledToZero = "ScaledToZero" +) + +// SandboxSetStatus is the observed state of the set. +type SandboxSetStatus struct { + // Replicas is how many Sandboxes the set currently owns, ready or not. It is the + // /scale subresource's status counterpart. + // +optional + Replicas int32 `json:"replicas,omitempty"` + + // ReadyReplicas is how many owned Sandboxes are Ready — the number of boxes that + // can actually serve an exec right now. + // +optional + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + + // Selector is the label selector matching this set's Sandboxes, serialized in the + // string form the /scale subresource requires. HPA and KEDA read the target's + // selector from there, so autoscaling a set does not work without it. + // +optional + Selector string `json:"selector,omitempty"` + + // Sandboxes names the boxes this set owns, so the set is a usable handout list: a + // caller reads it to find a box to use without listing and filtering Sandboxes + // itself. Ordered by name for a stable diff. + // +optional + Sandboxes []string `json:"sandboxes,omitempty"` + + // Conditions follows the standard Kubernetes condition convention. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=sbxs +// +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector +// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=`.spec.replicas` +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="NodePool",type=string,JSONPath=`.spec.template.spec.nodePoolRef` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// SandboxSet maintains N Sandboxes, so boxes can be kept ready ahead of demand +// instead of making a consumer wait minutes for an instance to provision. +type SandboxSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SandboxSetSpec `json:"spec,omitempty"` + Status SandboxSetStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// SandboxSetList contains a list of SandboxSet. +type SandboxSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SandboxSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&SandboxSet{}, &SandboxSetList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ffb03a9..2f20efa 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -290,3 +291,268 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Sandbox) DeepCopyInto(out *Sandbox) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Sandbox. +func (in *Sandbox) DeepCopy() *Sandbox { + if in == nil { + return nil + } + out := new(Sandbox) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Sandbox) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxList) DeepCopyInto(out *SandboxList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Sandbox, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxList. +func (in *SandboxList) DeepCopy() *SandboxList { + if in == nil { + return nil + } + out := new(SandboxList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSet) DeepCopyInto(out *SandboxSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSet. +func (in *SandboxSet) DeepCopy() *SandboxSet { + if in == nil { + return nil + } + out := new(SandboxSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSetList) DeepCopyInto(out *SandboxSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SandboxSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSetList. +func (in *SandboxSetList) DeepCopy() *SandboxSetList { + if in == nil { + return nil + } + out := new(SandboxSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SandboxSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSetSpec) DeepCopyInto(out *SandboxSetSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSetSpec. +func (in *SandboxSetSpec) DeepCopy() *SandboxSetSpec { + if in == nil { + return nil + } + out := new(SandboxSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSetStatus) DeepCopyInto(out *SandboxSetStatus) { + *out = *in + if in.Sandboxes != nil { + in, out := &in.Sandboxes, &out.Sandboxes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSetStatus. +func (in *SandboxSetStatus) DeepCopy() *SandboxSetStatus { + if in == nil { + return nil + } + out := new(SandboxSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxSpec) DeepCopyInto(out *SandboxSpec) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TTL != nil { + in, out := &in.TTL, &out.TTL + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxSpec. +func (in *SandboxSpec) DeepCopy() *SandboxSpec { + if in == nil { + return nil + } + out := new(SandboxSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxStatus) DeepCopyInto(out *SandboxStatus) { + *out = *in + if in.ReadyTime != nil { + in, out := &in.ReadyTime, &out.ReadyTime + *out = (*in).DeepCopy() + } + if in.ExpiryTime != nil { + in, out := &in.ExpiryTime, &out.ExpiryTime + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxStatus. +func (in *SandboxStatus) DeepCopy() *SandboxStatus { + if in == nil { + return nil + } + out := new(SandboxStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxTemplateMetadata) DeepCopyInto(out *SandboxTemplateMetadata) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxTemplateMetadata. +func (in *SandboxTemplateMetadata) DeepCopy() *SandboxTemplateMetadata { + if in == nil { + return nil + } + out := new(SandboxTemplateMetadata) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SandboxTemplateSpec) DeepCopyInto(out *SandboxTemplateSpec) { + *out = *in + in.Metadata.DeepCopyInto(&out.Metadata) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SandboxTemplateSpec. +func (in *SandboxTemplateSpec) DeepCopy() *SandboxTemplateSpec { + if in == nil { + return nil + } + out := new(SandboxTemplateSpec) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go index d70e105..82e3fff 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "flag" + "fmt" "os" "path/filepath" @@ -46,6 +47,7 @@ import ( nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" "github.com/InftyAI/Nebula/internal/controller" webhookv1 "github.com/InftyAI/Nebula/internal/webhook/v1" + nebulacert "github.com/InftyAI/Nebula/pkg/cert" "github.com/InftyAI/Nebula/pkg/failover" "github.com/InftyAI/Nebula/pkg/provider" awsprovider "github.com/InftyAI/Nebula/pkg/provider/aws" @@ -55,6 +57,10 @@ import ( // +kubebuilder:scaffold:imports ) +// defaultNamespace is where the manager is installed by config/default. It is only +// a fallback for managerNamespace when POD_NAMESPACE is unset. +const defaultNamespace = "nebula-system" + var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") @@ -216,6 +222,27 @@ func main() { os.Exit(1) } + // Provision the webhook serving cert in-process, replacing both cert-manager and + // the out-of-band hack/gen-webhook-cert.sh. The rotator mints the keypair into a + // Secret, writes it where the webhook server reads it, patches the caBundle into + // the MutatingWebhookConfiguration, and keeps renewing it before expiry — the one + // thing neither prior approach did (the script's cert simply expired years later). + // + // certsReady closes once the cert is on disk AND the caBundle is patched. The + // webhook must not register before that: with failurePolicy=Fail, serving on a + // missing keypair means every Pod CREATE in the cluster fails admission. + certsReady := make(chan struct{}) + if enableWebhooks() { + if err := nebulacert.CertsManager(mgr, managerNamespace(), certsReady); err != nil { + setupLog.Error(err, "unable to set up cert rotation") + os.Exit(1) + } + } else { + // Nothing will close the channel, so close it here or the goroutine below would + // block forever and no controller would ever start. + close(certsReady) + } + // Register provider backends into the process-wide registry that both // reconcilers resolve through (their Providers field defaults to // provider.Get). Done before SetupWithManager so a pool/claim reconciled at @@ -231,46 +258,27 @@ func main() { // both sides rather than persisted. blocklist := failover.New() - if err := (&controller.NodePoolReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NodePool") - os.Exit(1) - } - if err := (&controller.NodeClaimReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NodeClaim") - os.Exit(1) - } - if err := (&controller.PodPlacementReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Blocklist: blocklist, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "PodPlacement") - os.Exit(1) - } - - // Start one virtual node per registered provider. The virtual kubelet owns - // provisioning: its pod controller calls provider.Provision on CreatePod and - // provider.Terminate on DeletePod, so an ungated Pod bound to a provider's - // virtual node materializes an external instance. Each Runner is a - // manager.Runnable, so it shares the manager's lifecycle and leader election. - if err := setupVirtualNodes(mgr, blocklist); err != nil { - setupLog.Error(err, "unable to set up virtual nodes") - os.Exit(1) - } - // nolint:goconst - if os.Getenv("ENABLE_WEBHOOKS") != "false" { - if err := webhookv1.SetupPodWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Pod") + // Controller and webhook registration is deferred until the cert exists, so it + // runs in a goroutine: the cert cannot be minted until the manager is STARTED + // (the rotator is a Runnable and needs a synced cache), so blocking here would + // deadlock. controller-runtime supports Add after Start — a Runnable registered + // then is started immediately — which is what makes this safe. + // + // The controllers wait too, not just the webhook. They create Pods, and every Pod + // CREATE goes through the defaulting webhook that injects the provider-selection + // gate; a Pod created while that webhook is untrusted would either be rejected + // (failurePolicy=Fail) or, worse, admitted ungated and scheduled by vanilla + // Kubernetes — silently bypassing placement and never reaching a provider. + go func() { + setupLog.Info("waiting for the webhook certificate to be ready") + <-certsReady + setupLog.Info("webhook certificate ready") + + if err := setupControllers(mgr, blocklist); err != nil { + setupLog.Error(err, "unable to set up controllers") os.Exit(1) } - } - // +kubebuilder:scaffold:builder + }() if metricsCertWatcher != nil { setupLog.Info("Adding metrics certificate watcher to manager") @@ -304,6 +312,89 @@ func main() { } } +// enableWebhooks reports whether the Pod defaulting webhook (and therefore the cert +// rotator that makes it trustable) should run. It is off only when explicitly +// disabled, which is how the local `make run` and tests avoid needing a cert and a +// reachable Service. +func enableWebhooks() bool { + // nolint:goconst + return os.Getenv("ENABLE_WEBHOOKS") != "false" +} + +// managerNamespace is the namespace the manager runs in, which scopes both the +// webhook cert Secret and the cert's DNS name. It is read from POD_NAMESPACE +// (projected via fieldRef in config/manager/manager.yaml) rather than hardcoded, so +// an install into a non-default namespace still gets a cert the API server accepts. +// The fallback only matters for an out-of-cluster run, where the webhook is +// typically disabled anyway. +func managerNamespace() string { + if ns := os.Getenv("POD_NAMESPACE"); ns != "" { + return ns + } + setupLog.Info("POD_NAMESPACE is unset; falling back to the default install namespace", + "namespace", defaultNamespace) + return defaultNamespace +} + +// setupControllers registers every controller, the virtual nodes and the webhook. +// It runs only after the webhook serving cert is ready (see main), which is why it +// is a function rather than inline: everything here depends on Pod admission +// working, so none of it may be registered before the API server trusts the webhook. +func setupControllers(mgr ctrl.Manager, blocklist *failover.Blocklist) error { + if err := (&controller.NodePoolReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create NodePool controller: %w", err) + } + if err := (&controller.NodeClaimReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create NodeClaim controller: %w", err) + } + if err := (&controller.PodPlacementReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Blocklist: blocklist, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create PodPlacement controller: %w", err) + } + + // The workload controllers sit on top of the provisioning core above: each + // synthesizes objects onto the same placement path rather than talking to a + // provider itself. Sandbox produces the Pod that backs one remote box; + // SandboxSet produces Sandboxes. + if err := (&controller.SandboxReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create Sandbox controller: %w", err) + } + if err := (&controller.SandboxSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create SandboxSet controller: %w", err) + } + + // Start one virtual node per registered provider. The virtual kubelet owns + // provisioning: its pod controller calls provider.Provision on CreatePod and + // provider.Terminate on DeletePod, so an ungated Pod bound to a provider's + // virtual node materializes an external instance. Each Runner is a + // manager.Runnable, so it shares the manager's lifecycle and leader election. + if err := setupVirtualNodes(mgr, blocklist); err != nil { + return fmt.Errorf("unable to set up virtual nodes: %w", err) + } + if enableWebhooks() { + if err := webhookv1.SetupPodWebhookWithManager(mgr); err != nil { + return fmt.Errorf("unable to create Pod webhook: %w", err) + } + } + // +kubebuilder:scaffold:builder + return nil +} + // setupVirtualNodes adds a vnode.Runner to the manager for every registered // provider. The Runner needs a typed clientset (the virtual kubelet's node/pod // controllers use client-go directly, not the controller-runtime client), built diff --git a/config/crd/bases/nebula.inftyai.com_sandboxes.yaml b/config/crd/bases/nebula.inftyai.com_sandboxes.yaml new file mode 100644 index 0000000..e982561 --- /dev/null +++ b/config/crd/bases/nebula.inftyai.com_sandboxes.yaml @@ -0,0 +1,452 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: sandboxes.nebula.inftyai.com +spec: + group: nebula.inftyai.com + names: + kind: Sandbox + listKind: SandboxList + plural: sandboxes + shortNames: + - sbx + singular: sandbox + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.nodePoolRef + name: Pool + type: string + - jsonPath: .spec.acceleratorType + name: Accelerator + type: string + - jsonPath: .status.endpoint + name: Endpoint + type: string + - jsonPath: .status.expiryTime + name: Expires + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Sandbox is one interactive remote instance — an agent workspace, a shell, a + scratch GPU box — reachable with the same `kubectl exec` / `kubectl logs` a + local Pod would be. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + SandboxSpec is one long-lived, interactive remote box: an agent's workspace, a + shell, a scratch GPU machine. It is the first workload class Nebula serves + beyond a hand-written Pod. + + A Sandbox is SINGULAR — one object, one instance. There is deliberately no + replicas field and no pod template, because a sandbox is not fungible: someone + is attached to it, it accumulates state in its filesystem, and the object's own + name IS its stable identity (`kubectl exec sandbox-alice` always reaches the + same box). Set-shaped controllers exist because containers are interchangeable, + which is exactly the property a sandbox lacks — a rolling update would evict a + live session, and "scale in by one" would have to guess whose box to kill. A + caller that wants N boxes creates N Sandboxes, each with its own image, + lifetime and identity. + + The count lives one level up, in SandboxSet, which maintains N Sandboxes and + owns /scale so `kubectl scale` and HPA work. That split is what lets this type + stay singular: because the set creates Sandbox OBJECTS rather than replicas + inside one object, everything that depends on a box being its own object — + per-box RBAC (grant a user their sandbox and not their neighbour's), a + per-box image and TTL, and a failure that stays visible instead of being + papered over by a replacement — keeps working underneath a pool. + + The spec deliberately reuses corev1 types (ResourceRequirements, EnvVar) rather + than inventing parallel fields. The controller synthesizes a Pod, so anything + it accepts must ultimately BE PodSpec-shaped; re-declaring resources or env + would fork the vocabulary and, worse, fork the source of truth for the + accelerator COUNT — which placement and the scheduler's fit check both read + from the container's nvidia.com/gpu limit (see util.AcceleratorRequest). + + The CEL rule below rejects a GPU count with no accelerator type. That pair is + contradictory rather than merely incomplete — util.AcceleratorRequest returns an + error for it — so without the rule the object is admitted and then fails at + PLACEMENT, minutes later and one object removed from the mistake. Note the + inverse is fine and deliberately allowed: a type with no count means one + accelerator. + properties: + acceleratorType: + description: |- + AcceleratorType is the requested accelerator TYPE (e.g. "a100-40gb", + "h100"), matched case-insensitively against the provider catalog. The COUNT + is NOT here: it is a standard nvidia.com/gpu entry in Resources, so exactly + one number drives scheduling fit and provisioning. The controller stamps + this onto the synthesized Pod's AcceleratorTypeLabel, so a Sandbox and a + hand-written Nebula Pod go through identical placement. + + Empty means a CPU-only sandbox, which is a legitimate (and cheap) thing to + want for a shell or an agent that only needs a filesystem. + type: string + env: + description: |- + Env is passed to the sandbox container verbatim, including valueFrom + references — a sandbox usually needs at least a registry or Hugging Face + token, and re-inventing secret indirection here would be strictly worse than + reusing the field everyone already knows. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + default: ubuntu:24.04 + description: |- + Image is the container image the sandbox runs. It defaults to a plain Ubuntu, + because unlike the accelerator the image is not a decision a caller has to make + to get a useful box: `kubectl exec` into a bare distro is exactly the "give me a + remote shell" case, and anything else can be installed from inside it. Defaulting + a paid GPU shape would be guessing at spend; defaulting a shell is not. + + Note it deliberately does NOT default to a CUDA image even when an accelerator is + requested. A conditional default would make the image depend on another field, + which structural-schema defaulting cannot express and which would surprise anyone + reading the object back. Ask for a CUDA image explicitly when you want one. + + There is deliberately no command field, and one cannot be set: the CRD is a + structural schema, so `command:` in a Sandbox spec is rejected as an unknown + field by the apiserver itself — no webhook required. That is not a + simplification, it is the process model: the container's command is always + SandD, which runs as PID 1 and is what makes `kubectl exec` and `kubectl logs` + work against an instance in another cloud. A user-supplied command would + displace it and take both with it. + + A sandbox has nothing to run at boot anyway — the whole point is that commands + arrive later, over exec — so SandD spawns no child here; it holds the container + open and serves requests. Workload classes that do run something get it spawned + as SandD's child instead, which is how it comes to own their stdout/stderr. + minLength: 1 + type: string + nodePoolRef: + description: |- + NodePoolRef names the NodePool whose policy places this sandbox: which + providers are allowed, which capacity tiers, how to rank them. Required — + there is no implicit default pool, because placing a paid GPU instance + against a guessed policy is not a safe default. + minLength: 1 + type: string + resources: + description: |- + Resources is the standard Kubernetes resource requirements for the sandbox + container, verbatim. The accelerator count rides here as an nvidia.com/gpu + limit (`limits: {nvidia.com/gpu: "1"}`), which is where both the placement + controller and the scheduler already read it from. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + ttl: + description: |- + TTL bounds the sandbox's total lifetime, measured from the moment it first + became Ready (NOT from creation, so a slow provision does not eat into the + user's time). On expiry the controller releases the instance and the sandbox + reports phase Expired. + + This exists because the failure mode of a remote GPU box is financial: an + abandoned sandbox bills until someone notices. Omit it for an unbounded + sandbox, which is a deliberate choice rather than the default. + type: string + required: + - nodePoolRef + type: object + x-kubernetes-validations: + - message: nvidia.com/gpu requires acceleratorType to be set + rule: has(self.acceleratorType) || !has(self.resources) || ((!has(self.resources.limits) + || !('nvidia.com/gpu' in self.resources.limits)) && (!has(self.resources.requests) + || !('nvidia.com/gpu' in self.resources.requests))) + status: + description: SandboxStatus is the observed state, projected from the synthesized + Pod. + properties: + conditions: + description: Conditions follows the standard Kubernetes condition + convention. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + endpoint: + description: |- + Endpoint is the reachable address of the external instance once it is + running, in the provider's own form (a public DNS name or an IP). Mirrored + from the Pod's EndpointAnnotation, which is where the virtual kubelet + publishes it. + type: string + expiryTime: + description: |- + ExpiryTime is when TTL will elapse (ReadyTime + TTL), surfaced so a user can + see the deadline without doing the arithmetic. Absent when no TTL is set or + the sandbox has not become Ready yet. + format: date-time + type: string + phase: + description: Phase is the coarse lifecycle state. + type: string + podName: + description: |- + PodName is the synthesized Pod backing this sandbox. It is recorded even + though it currently equals the Sandbox name, so tooling (and `kubectl exec` + wrappers) read the pod identity from status rather than reconstructing it + from a naming convention this controller would then be unable to change. + type: string + readyTime: + description: |- + ReadyTime is when the sandbox first became Ready. It is the anchor TTL is + measured from, so it is durable status rather than a derived value: if it + were recomputed from the Pod, a Pod status blip could silently restart the + user's clock. + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/nebula.inftyai.com_sandboxsets.yaml b/config/crd/bases/nebula.inftyai.com_sandboxsets.yaml new file mode 100644 index 0000000..dfc1e17 --- /dev/null +++ b/config/crd/bases/nebula.inftyai.com_sandboxsets.yaml @@ -0,0 +1,479 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: sandboxsets.nebula.inftyai.com +spec: + group: nebula.inftyai.com + names: + kind: SandboxSet + listKind: SandboxSetList + plural: sandboxsets + shortNames: + - sbxs + singular: sandboxset + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .spec.template.spec.nodePoolRef + name: NodePool + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxSet maintains N Sandboxes, so boxes can be kept ready ahead of demand + instead of making a consumer wait minutes for an instance to provision. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + SandboxSetSpec maintains N Sandboxes. That is the whole contract, and the name + says exactly that much: it is a SET, not a pool. A pool would imply lease + semantics — claim a box, hold it, return it, with the pool tracking who has what + — and none of that is implemented here. Keeping N boxes alive is what ENABLES + warm pooling (holding instances ready because provisioning takes minutes while + an agent's exec call wants sub-second) and fan-out ("twenty boxes for this + batch"), but those are uses of a set, not the set's job. "Pool" would also be a + third meaning of that word in this API group, where NodePool already means + placement policy. + + It creates Sandbox OBJECTS, not replicas inside itself, and that is the point of + having two types. The count is a genuinely different concern from the box: a set + answers "how many", a Sandbox answers "which one, running what, for whom". + Because each box stays its own object underneath, per-box RBAC, per-box status + and a failure that stays visible all keep working — none of which survives being + flattened into a replica index. + + Boxes get GENERATED names (myset-a4f2x), not ordinals. Ordinals would imply a + slot that gets refilled, so a box that died would be replaced by an empty one + wearing the same name — the same address with a different filesystem, which is + the most confusing thing this API could do. A generated name means a replacement + is visibly a NEW box, and callers that need a stable handle hold the Sandbox name + they were given rather than an index into a set. + properties: + replicas: + default: 1 + description: |- + Replicas is how many Sandboxes to maintain. Zero is legal and useful: it + releases every box while keeping the set's definition, which is how a set is + parked overnight without being forgotten. + format: int32 + minimum: 0 + type: integer + template: + description: |- + Template is the shape of every Sandbox this set creates. All boxes in one set + are the same shape by construction — the set exists to make boxes + interchangeable at the point of HANDOUT, so a caller can take any ready box + without inspecting it. Two shapes means two sets. + + A template is right here for the same reason it is wrong on Sandbox itself: + this object does not describe a box, it describes how to make them. + properties: + metadata: + description: |- + Metadata is the labels and annotations applied to each created Sandbox. Only + labels and annotations are honoured; a name here is ignored, since names are + generated per box. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are applied to each created Sandbox. + type: object + labels: + additionalProperties: + type: string + description: |- + Labels are applied to each created Sandbox, on top of the set-ownership + labels the controller adds. + type: object + type: object + spec: + description: Spec is the SandboxSpec of every box in the set. + properties: + acceleratorType: + description: |- + AcceleratorType is the requested accelerator TYPE (e.g. "a100-40gb", + "h100"), matched case-insensitively against the provider catalog. The COUNT + is NOT here: it is a standard nvidia.com/gpu entry in Resources, so exactly + one number drives scheduling fit and provisioning. The controller stamps + this onto the synthesized Pod's AcceleratorTypeLabel, so a Sandbox and a + hand-written Nebula Pod go through identical placement. + + Empty means a CPU-only sandbox, which is a legitimate (and cheap) thing to + want for a shell or an agent that only needs a filesystem. + type: string + env: + description: |- + Env is passed to the sandbox container verbatim, including valueFrom + references — a sandbox usually needs at least a registry or Hugging Face + token, and re-inventing secret indirection here would be strictly worse than + reusing the field everyone already knows. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + default: ubuntu:24.04 + description: |- + Image is the container image the sandbox runs. It defaults to a plain Ubuntu, + because unlike the accelerator the image is not a decision a caller has to make + to get a useful box: `kubectl exec` into a bare distro is exactly the "give me a + remote shell" case, and anything else can be installed from inside it. Defaulting + a paid GPU shape would be guessing at spend; defaulting a shell is not. + + Note it deliberately does NOT default to a CUDA image even when an accelerator is + requested. A conditional default would make the image depend on another field, + which structural-schema defaulting cannot express and which would surprise anyone + reading the object back. Ask for a CUDA image explicitly when you want one. + + There is deliberately no command field, and one cannot be set: the CRD is a + structural schema, so `command:` in a Sandbox spec is rejected as an unknown + field by the apiserver itself — no webhook required. That is not a + simplification, it is the process model: the container's command is always + SandD, which runs as PID 1 and is what makes `kubectl exec` and `kubectl logs` + work against an instance in another cloud. A user-supplied command would + displace it and take both with it. + + A sandbox has nothing to run at boot anyway — the whole point is that commands + arrive later, over exec — so SandD spawns no child here; it holds the container + open and serves requests. Workload classes that do run something get it spawned + as SandD's child instead, which is how it comes to own their stdout/stderr. + minLength: 1 + type: string + nodePoolRef: + description: |- + NodePoolRef names the NodePool whose policy places this sandbox: which + providers are allowed, which capacity tiers, how to rank them. Required — + there is no implicit default pool, because placing a paid GPU instance + against a guessed policy is not a safe default. + minLength: 1 + type: string + resources: + description: |- + Resources is the standard Kubernetes resource requirements for the sandbox + container, verbatim. The accelerator count rides here as an nvidia.com/gpu + limit (`limits: {nvidia.com/gpu: "1"}`), which is where both the placement + controller and the scheduler already read it from. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + ttl: + description: |- + TTL bounds the sandbox's total lifetime, measured from the moment it first + became Ready (NOT from creation, so a slow provision does not eat into the + user's time). On expiry the controller releases the instance and the sandbox + reports phase Expired. + + This exists because the failure mode of a remote GPU box is financial: an + abandoned sandbox bills until someone notices. Omit it for an unbounded + sandbox, which is a deliberate choice rather than the default. + type: string + required: + - nodePoolRef + type: object + x-kubernetes-validations: + - message: nvidia.com/gpu requires acceleratorType to be set + rule: has(self.acceleratorType) || !has(self.resources) || ((!has(self.resources.limits) + || !('nvidia.com/gpu' in self.resources.limits)) && (!has(self.resources.requests) + || !('nvidia.com/gpu' in self.resources.requests))) + required: + - spec + type: object + required: + - template + type: object + status: + description: SandboxSetStatus is the observed state of the set. + properties: + conditions: + description: Conditions follows the standard Kubernetes condition + convention. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + readyReplicas: + description: |- + ReadyReplicas is how many owned Sandboxes are Ready — the number of boxes that + can actually serve an exec right now. + format: int32 + type: integer + replicas: + description: |- + Replicas is how many Sandboxes the set currently owns, ready or not. It is the + /scale subresource's status counterpart. + format: int32 + type: integer + sandboxes: + description: |- + Sandboxes names the boxes this set owns, so the set is a usable handout list: a + caller reads it to find a box to use without listing and filtering Sandboxes + itself. Ordered by name for a stable diff. + items: + type: string + type: array + selector: + description: |- + Selector is the label selector matching this set's Sandboxes, serialized in the + string form the /scale subresource requires. HPA and KEDA read the target's + selector from there, so autoscaling a set does not work without it. + type: string + type: object + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index b8628ac..73b1328 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,8 @@ resources: - bases/nebula.inftyai.com_nodepools.yaml - bases/nebula.inftyai.com_nodeclaims.yaml +- bases/nebula.inftyai.com_sandboxes.yaml +- bases/nebula.inftyai.com_sandboxsets.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 6e42628..f169c70 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -24,11 +24,13 @@ resources: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - ../webhook -# [CERTMANAGER] cert-manager is intentionally NOT used. The webhook serving -# cert is generated as a self-signed cert by hack/gen-webhook-cert.sh (run via -# `make deploy-all`), which also injects the CA into the webhook config. To -# switch back to cert-manager, re-add `- ../certmanager` here and re-enable the -# CERTMANAGER replacements blocks below. See docs/deploy.md. +# [CERTMANAGER] cert-manager is intentionally NOT used, and neither is any +# out-of-band cert step. The manager provisions its own webhook serving cert +# in-process at startup (pkg/cert): it mints the keypair into a Secret, writes it +# to disk, patches the caBundle into the webhook config, and RENEWS it before +# expiry. To switch to cert-manager, re-add `- ../certmanager` here, re-enable the +# CERTMANAGER replacements blocks below, and drop the CertsManager call from +# cmd/main.go. See docs/deploy.md. #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -47,9 +49,11 @@ patches: - path: manager_metrics_patch.yaml target: kind: Deployment - # Scope to the manager ONLY (matches the pre-prefix name), so the - # manager-only --metrics-bind-address flag is not injected into any other - # Deployment in the build. + # Scope to the manager ONLY (matches the pre-prefix name). The manager is + # currently the only Deployment in the build, so this is defensive rather than + # load-bearing: without a name, kustomize applies the patch to EVERY Deployment, + # so the day an overlay adds a second one it would silently receive the + # manager-only --metrics-bind-address flag and fail to start on an unknown flag. name: controller-manager # Uncomment the patches line if you enable Metrics and CertManager @@ -64,7 +68,8 @@ patches: - path: manager_webhook_patch.yaml target: kind: Deployment - # Scope to the manager ONLY (see the metrics patch above). + # Scope to the manager ONLY (see the metrics patch above) — otherwise a second + # Deployment added later would also get the webhook port, volume and cert mount. name: controller-manager # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. @@ -130,7 +135,7 @@ replacements: # index: 1 # create: true -# [CERTMANAGER] Disabled — self-signed cert via hack/gen-webhook-cert.sh instead. +# [CERTMANAGER] Disabled — the cert is minted in-process by pkg/cert instead. # - source: # Uncomment the following block if you have any webhook # kind: Service # version: v1 @@ -199,8 +204,8 @@ replacements: # index: 1 # create: true -# [CERTMANAGER] Disabled — the CA is injected into the MutatingWebhookConfiguration -# by hack/gen-webhook-cert.sh (kubectl patch caBundle) instead of this annotation. +# [CERTMANAGER] Disabled — the CA is patched into the MutatingWebhookConfiguration +# by the in-process cert rotator (pkg/cert) instead of this annotation. # - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) # kind: Certificate # group: cert-manager.io diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml index 963c8a4..cbabe0c 100644 --- a/config/default/manager_webhook_patch.yaml +++ b/config/default/manager_webhook_patch.yaml @@ -1,12 +1,40 @@ -# This patch ensures the webhook certificates are properly mounted in the manager container. -# It configures the necessary arguments, volumes, volume mounts, and container ports. +# This patch wires up the webhook server: its port, and the directory it serves its +# TLS keypair from. +# +# The keypair is provisioned IN-PROCESS by the cert rotator (pkg/cert), not by +# cert-manager and not by an out-of-band script. Two consequences worth spelling out, +# because they are easy to get backwards: +# +# 1. The keypair reaches this directory as a SECRET PROJECTION, not an emptyDir. +# The rotator does not write files: it mints the keypair into the Secret and +# nothing else — cert-controller has no disk writes at all, and its CertDir is a +# path it only ever os.Stat()s to decide readiness (see ensureCertsMounted in +# rotator.go). So the kubelet is what puts the keypair on disk. With an emptyDir +# the files would never appear, the rotator's IsReady channel would never close, +# and — because controller and webhook registration waits on it (cmd/main.go) — +# NOTHING would ever start: no reconcilers, no virtual nodes, no webhook. The +# manager stays Running and looks healthy, which is what makes that failure mode +# nasty. This is also the second reason config/webhook ships the empty Secret: +# a non-optional Secret volume blocks the pod from starting until it exists. +# +# 2. There is no --webhook-cert-path argument. Passing it makes main build a +# certwatcher over the path, and the watcher fails at STARTUP if the files are not +# there yet — which they are not on a first install, because the Secret is empty +# until the rotator mints into it. controller-runtime's webhook server already +# reads this exact directory by default, and it is only registered after the +# rotator reports ready, so by then the files are there. -# Add the --webhook-cert-path argument for configuring the webhook certificate path +# Add the port configuration for the webhook server - op: add - path: /spec/template/spec/containers/0/args/- - value: --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs + path: /spec/template/spec/containers/0/ports/- + value: + containerPort: 9443 + name: webhook-server + protocol: TCP -# Add the volumeMount for the webhook certificates +# The directory the keypair is projected into and the webhook server serves from. +# Must match certDir in pkg/cert (controller-runtime's default path), which is also +# the path the rotator polls to decide readiness. - op: add path: /spec/template/spec/containers/0/volumeMounts/- value: @@ -14,18 +42,12 @@ name: webhook-certs readOnly: true -# Add the port configuration for the webhook server -- op: add - path: /spec/template/spec/containers/0/ports/- - value: - containerPort: 9443 - name: webhook-server - protocol: TCP - -# Add the volume configuration for the webhook certificates +# The rotator's Secret, projected read-only (see above: the kubelet, not the rotator, +# writes these files). Name must match secretName in pkg/cert. NOT optional: the pod +# should refuse to start rather than run with a directory that never fills. - op: add path: /spec/template/spec/volumes/- value: name: webhook-certs secret: - secretName: webhook-server-cert + secretName: nebula-webhook-server-cert diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index f8b02c0..444da50 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -72,6 +72,15 @@ spec: # embedded in the binary. - name: NEBULA_CATALOG_DIR value: /etc/nebula/catalog + # The namespace the manager runs in, needed by the webhook cert rotator + # (pkg/cert): it scopes both the cert Secret and the cert's DNS name + # (..svc). Projected via fieldRef rather than hardcoded + # so an install into a non-default namespace still gets a cert the API + # server accepts. + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace envFrom: # Provider credentials live in a per-provider Secret, one secretRef per # provider — NOT a single shared secret. This matches the "creds-absent → diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 201b154..6c6b589 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -8,7 +8,6 @@ rules: - "" resources: - configmaps - - secrets - services verbs: - get @@ -25,6 +24,7 @@ rules: - "" resources: - nodes + - pods verbs: - create - delete @@ -45,12 +45,20 @@ rules: - apiGroups: - "" resources: - - pods + - secrets + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations verbs: - - delete - get - list - - patch - update - watch - apiGroups: @@ -70,6 +78,8 @@ rules: resources: - nodeclaims - nodepools + - sandboxes + - sandboxsets verbs: - create - delete @@ -83,6 +93,8 @@ rules: resources: - nodeclaims/finalizers - nodepools/finalizers + - sandboxes/finalizers + - sandboxsets/finalizers verbs: - update - apiGroups: @@ -90,6 +102,9 @@ rules: resources: - nodeclaims/status - nodepools/status + - sandboxes/status + - sandboxsets/scale + - sandboxsets/status verbs: - get - patch diff --git a/config/samples/nebula_v1alpha1_sandbox.yaml b/config/samples/nebula_v1alpha1_sandbox.yaml new file mode 100644 index 0000000..b8b3436 --- /dev/null +++ b/config/samples/nebula_v1alpha1_sandbox.yaml @@ -0,0 +1,33 @@ +# One interactive remote box. Reach it exactly as you would a local Pod: +# +# kubectl exec -it sample -- bash +# kubectl logs sample +# +# There is no command field, by design: the container's command is always SandD, +# which runs as PID 1 and is what makes exec and logs work against an instance in +# another cloud. A box has nothing to run at boot anyway — commands arrive later, +# over exec — so SandD holds the container open and serves them. +apiVersion: nebula.inftyai.com/v1alpha1 +kind: Sandbox +metadata: + labels: + app.kubernetes.io/managed-by: nebula + name: sample +spec: + # The placement policy this box is provisioned against: which providers are + # allowed, which capacity tiers, how to rank them. + nodePoolRef: sample + # Optional — defaults to ubuntu:24.04, which is all a "give me a remote shell" box + # needs. Note it does NOT become a CUDA image when you ask for an accelerator; name + # one explicitly if you want the toolkit preinstalled. + image: ubuntu:24.04 + # The accelerator TYPE is a label-shaped field; the COUNT rides on the + # nvidia.com/gpu resource below, so exactly one number drives both scheduling + # fit and provisioning. Omit both for a CPU-only box. + acceleratorType: a100-40gb + resources: + requests: + cpu: "8" + memory: 64Gi + limits: + nvidia.com/gpu: "1" diff --git a/config/samples/nebula_v1alpha1_sandboxset.yaml b/config/samples/nebula_v1alpha1_sandboxset.yaml new file mode 100644 index 0000000..481ca4f --- /dev/null +++ b/config/samples/nebula_v1alpha1_sandboxset.yaml @@ -0,0 +1,36 @@ +# Maintains N Sandboxes. Provisioning an instance takes MINUTES while an agent's +# exec call wants sub-second, so boxes are kept ready ahead of demand: +# +# kubectl get sandboxes -l nebula.inftyai.com/sandboxset=sample +# kubectl scale sandboxset/sample --replicas=5 +# +# Boxes get generated names (sample-a4f2x), not ordinals, because an ordinal would +# imply a slot that gets refilled — a replacement box wearing a dead box's name is +# the same address with a different filesystem. A generated name makes a +# replacement visibly a new box. +# +# A template change does NOT roll existing boxes (this is ReplicaSet-shaped, not +# Deployment-shaped): rolling would evict live sessions and burn minutes of +# provisioning per box. New boxes get the new template; existing ones are left alone. +apiVersion: nebula.inftyai.com/v1alpha1 +kind: SandboxSet +metadata: + labels: + app.kubernetes.io/managed-by: nebula + name: sample +spec: + replicas: 3 + template: + metadata: + labels: + team: ml + spec: + nodePoolRef: sample + image: ubuntu:24.04 + acceleratorType: a100-40gb + resources: + requests: + cpu: "8" + memory: 64Gi + limits: + nvidia.com/gpu: "1" diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 051676b..e74dbed 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,6 +1,9 @@ resources: - manifests.yaml - service.yaml +# The (empty) Secret the rotator populates. Required — the rotator Gets/Updates it +# but never Creates it, so without this the manager crash-loops. See secret.yaml. +- secret.yaml # Narrow the generated Pod webhook to opt-in Pods outside system namespaces. patches: diff --git a/config/webhook/secret.yaml b/config/webhook/secret.yaml new file mode 100644 index 0000000..35bf863 --- /dev/null +++ b/config/webhook/secret.yaml @@ -0,0 +1,18 @@ +# The Secret the webhook serving keypair lives in. +# +# It is created EMPTY on purpose, and that is not a placeholder — it is required. +# The cert rotator (pkg/cert) does a Get followed by an Update on this Secret and +# never a Create, so an absent Secret is a fatal startup error ("acquiring secret to +# update certificates: not found") that crash-loops the manager. An empty Secret is +# exactly what it expects: `secret.Data == nil` is the trigger for minting the CA and +# server certs and writing them back here. +# +# So the contents are owned entirely by the manager at runtime — never commit key +# material, and do not hand-populate it. It is also the shared source of truth across +# replicas: the first pod to run mints the keypair into this Secret, and later pods +# find it already valid and just write it to their own local disk. +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: system diff --git a/docs/architecture.md b/docs/architecture.md index 463c4b4..fddafac 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -614,25 +614,3 @@ manager owns reconciliation and the VK node leases (the VK nodes run inside the manager as `Runnable`s — option A). --- - -## Build status - -| Component | Package | Status | -|---|---|---| -| API types (NodePool, NodeClaim) | `api/v1alpha1` | DONE | -| Provider interface + registry | `pkg/provider` | DONE | -| Price catalog (CSV + ConfigMap) | `pkg/provider/catalog` | DONE | -| Modal adapter | `pkg/provider/modal` | DONE | -| NodeClaim controller (teardown backstop) | `internal/controller` | DONE | -| NodePool controller | `internal/controller` | DONE | -| Scheduling-gate webhook | `internal/webhook/v1` | DONE | -| Provider wiring in manager | `cmd/main.go` | DONE | -| Virtual Kubelet node (VK owns provisioning) | `pkg/vnode` | DONE | -| Placement controller (first-matching-provider) | `internal/controller` | DONE | -| **Optimizer (price/weighted, capacity fallback, blocklist)** | — | **PLANNED** | -| Other adapters (RunPod next) | `pkg/provider/*` | PLANNED | - -The critical path — gated Pod → placed → bound → provisioned → torn down — is now -closed end to end. The next milestone is the **optimizer**: replacing the v1 -"first matching provider" policy behind `selectPlacement` with price/weighted -ranking, capacity-tier fallback, and the failover blocklist. diff --git a/docs/deploy.md b/docs/deploy.md index c2bc8a5..c25de13 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -68,40 +68,46 @@ cluster) is passed as `make` variables, never committed to `.env`. Nebula runs a mutating webhook (it injects the scheduling gate into gated Pods), and the API server requires TLS to call it. **cert-manager is intentionally not -used** — there is no cert-manager prerequisite to install. Instead, -`hack/gen-webhook-cert.sh` provisions a self-signed certificate. It does the two -things cert-manager would otherwise automate: - -1. **Serving cert** — generates a self-signed cert for the webhook Service DNS - name (`nebula-webhook-service.nebula-system.svc`) and stores it in the - `webhook-server-cert` TLS Secret the manager mounts. -2. **CA trust** — injects that cert's CA into the `MutatingWebhookConfiguration` - `caBundle`, so the API server trusts the webhook. The caBundle is always read - back *from the Secret*, so the served cert and the trusted CA cannot drift. - -**No manager restart is involved.** `deploy-all` orders things so the manager -boots already-correct: the cert Secret is created *before* `make deploy` (it is a -required volume mount, and a running pod would otherwise need remounting), while -the caBundle patch happens *after* (it edits only the webhook config, which only -the API server reads — the manager never sees it). - -Re-running is safe: an existing cert Secret is kept as-is, and the caBundle is -re-derived to match. - -> **No auto-rotation.** This is the one thing cert-manager gives you that the -> self-signed cert does not. The cert is valid 10 years (`CERT_DAYS`, default -> `3650`). To rotate/renew, regenerate the cert and re-inject the CA: -> ```bash -> FORCE_REGEN=true hack/gen-webhook-cert.sh secret -> hack/gen-webhook-cert.sh cabundle -> kubectl rollout restart deployment/nebula-controller-manager -n nebula-system -> ``` -> The restart here is only because you are *replacing* the cert under a -> already-running manager — the initial deploy needs none. - -To switch back to cert-manager, re-add `- ../certmanager` and re-enable the +used**, and neither is any out-of-band setup step: the manager provisions its own +serving certificate in-process at startup (`pkg/cert`, built on +[cert-controller](https://github.com/open-policy-agent/cert-controller)). There is +nothing to install and nothing to run before `make deploy`. + +It does the three things that have to agree with each other: + +1. **Serving cert** — mints a self-signed cert for the webhook Service DNS name + (`nebula-webhook-service..svc`) and stores it in the + `nebula-webhook-server-cert` Secret. That Secret is projected into the manager at + `/tmp/k8s-webhook-server/serving-certs`, where the webhook server reads it — the + rotator writes only the Secret, never the filesystem, so the volume at that path + must be the Secret rather than an `emptyDir`. +2. **CA trust** — patches that cert's CA into the `MutatingWebhookConfiguration` + `caBundle`, so the API server trusts the webhook. It is derived from the cert + just written, so the served cert and the trusted CA cannot drift. +3. **Renewal** — rotates the cert before it expires. This is the part neither + cert-manager-free alternative had: a script-minted cert simply expires, years + later, when nobody remembers a script was involved. + +The Secret is the shared source of truth across replicas — a second replica finds +the existing cert there rather than minting a competing one, and the kubelet projects +it into that pod too. Rotation is *not* leader-elected, because webhook serving is not +either: every replica needs the keypair on its own local disk, and the API server +will call a non-leader. + +**Startup ordering.** Nothing that depends on Pod admission is registered until the +cert is ready, so the first seconds of a fresh install log +`waiting for the webhook certificate to be ready` and reconcile nothing. That is +deliberate: with `failurePolicy: Fail` a Pod created before the webhook is trusted +would be rejected, and without the gate it would be scheduled by vanilla Kubernetes +— silently bypassing placement and never reaching a provider. + +The cert volume is an `emptyDir`, not a Secret projection, because the rotator +*writes* to that path; a Secret volume is read-only and its kubelet refresh would +fight the rotator. + +To switch to cert-manager instead, re-add `- ../certmanager` and re-enable the `CERTMANAGER` replacements blocks in `config/default/kustomization.yaml`, install -cert-manager, and drop the `gen-webhook-cert.sh` calls from `hack/deploy.sh`. +cert-manager, and drop the `CertsManager` call from `cmd/main.go`. --- @@ -151,23 +157,20 @@ Non-secret config, passed as `make` variables: If you don't want the script (e.g. you manage Secrets via sealed-secrets or a GitOps pipeline), do the same steps by hand. Order matters — create the Secrets -before deploying so the manager boots configured, and inject the CA after: +before deploying so the manager boots configured: ```bash -# 1. Namespace + webhook serving cert Secret (before deploy — it's a volume mount). +# 1. Namespace. kubectl create namespace nebula-system --dry-run=client -o yaml | kubectl apply -f - -hack/gen-webhook-cert.sh secret # 2. Modal credential Secret (before deploy — read as env at pod startup). kubectl create secret generic nebula-modal-credentials -n nebula-system \ --from-literal=MODAL_TOKEN_ID=ak-... \ --from-literal=MODAL_TOKEN_SECRET=as-... -# 3. Deploy CRDs + manager. The pod mounts the cert and reads creds on first boot. +# 3. Deploy CRDs + manager. The pod reads creds on first boot, and provisions its +# own webhook cert + caBundle at startup — no cert step of your own. make deploy IMG=/nebula: - -# 4. Inject the webhook CA (server-side; needs the webhook config to exist). -hack/gen-webhook-cert.sh cabundle ``` No restart is needed — everything the manager consumes exists before it boots. @@ -201,7 +204,7 @@ kubectl -n nebula-system logs deploy/nebula-controller-manager | grep -i provide kubectl get nodes -l nebula.inftyai.com/provider # Webhook TLS is wired: the caBundle matches the serving cert Secret. -diff <(kubectl get secret webhook-server-cert -n nebula-system -o jsonpath='{.data.tls\.crt}') \ +diff <(kubectl get secret nebula-webhook-server-cert -n nebula-system -o jsonpath='{.data.tls\.crt}') \ <(kubectl get mutatingwebhookconfiguration nebula-mutating-webhook-configuration \ -o jsonpath='{.webhooks[0].clientConfig.caBundle}') \ && echo "webhook caBundle matches serving cert" diff --git a/go.mod b/go.mod index 6b222b3..77c3fb7 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,12 @@ require ( github.com/modal-labs/modal-client/go v0.9.0 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 + github.com/open-policy-agent/cert-controller v0.14.0 github.com/prometheus/client_model v0.6.1 github.com/virtual-kubelet/virtual-kubelet v1.11.0 - k8s.io/api v0.33.3 - k8s.io/apimachinery v0.33.3 - k8s.io/client-go v0.33.3 + k8s.io/api v0.33.4 + k8s.io/apimachinery v0.33.4 + k8s.io/client-go v0.33.4 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 sigs.k8s.io/controller-runtime v0.21.0 ) @@ -42,7 +43,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/djherbis/buffer v1.2.0 // indirect github.com/djherbis/nio/v3 v3.0.1 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -51,7 +52,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -94,6 +95,7 @@ require ( go.opentelemetry.io/otel/sdk v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -115,9 +117,9 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.33.0 // indirect - k8s.io/apiserver v0.33.0 // indirect - k8s.io/component-base v0.33.0 // indirect + k8s.io/apiextensions-apiserver v0.33.4 // indirect + k8s.io/apiserver v0.33.4 // indirect + k8s.io/component-base v0.33.4 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect diff --git a/go.sum b/go.sum index 83be65a..7ce5be5 100644 --- a/go.sum +++ b/go.sum @@ -54,7 +54,6 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -64,8 +63,8 @@ github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= +github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -93,12 +92,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -166,11 +163,8 @@ github.com/kisielk/og-rek v1.3.0 h1:lTXdQXqFETZKA//FWH4RBNAuiJ/dofxIwHAidoUZoMk= github.com/kisielk/og-rek v1.3.0/go.mod h1:4at7oxyfBTDilURhNCf7irHWtosJlJl9uyqUqAkrP4w= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -198,6 +192,10 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/open-policy-agent/cert-controller v0.14.0 h1:TPc19BOHOs4tARruTT5o4bzir7Ed6FF+j3EXP/nmZBs= +github.com/open-policy-agent/cert-controller v0.14.0/go.mod h1:UhE/FU54DnKo+Rt0Yf3r+oKjgy6kqSH8Vsjo+5bGrSo= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a h1:gQtOJ50XFyL2Xh3lDD9zP4KQ2PY4mZKQ9hDcWc81Sp8= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a/go.mod h1:tI7nc6H6os2UYZRvSm9Y7bq4oMoXqhwA0WfnqKpoAgc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -272,6 +270,8 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -387,20 +387,22 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= -k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= -k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= -k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= -k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= -k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= -k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= -k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= -k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= -k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk= -k8s.io/component-base v0.33.0/go.mod h1:aXYZLbw3kihdkOPMDhWbjGCO6sg+luw554KP51t8qCU= +k8s.io/api v0.33.4 h1:oTzrFVNPXBjMu0IlpA2eDDIU49jsuEorGHB4cvKupkk= +k8s.io/api v0.33.4/go.mod h1:VHQZ4cuxQ9sCUMESJV5+Fe8bGnqAARZ08tSTdHWfeAc= +k8s.io/apiextensions-apiserver v0.33.4 h1:rtq5SeXiDbXmSwxsF0MLe2Mtv3SwprA6wp+5qh/CrOU= +k8s.io/apiextensions-apiserver v0.33.4/go.mod h1:mWXcZQkQV1GQyxeIjYApuqsn/081hhXPZwZ2URuJeSs= +k8s.io/apimachinery v0.33.4 h1:SOf/JW33TP0eppJMkIgQ+L6atlDiP/090oaX0y9pd9s= +k8s.io/apimachinery v0.33.4/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.4 h1:6N0TEVA6kASUS3owYDIFJjUH6lgN8ogQmzZvaFFj1/Y= +k8s.io/apiserver v0.33.4/go.mod h1:8ODgXMnOoSPLMUg1aAzMFx+7wTJM+URil+INjbTZCok= +k8s.io/client-go v0.33.4 h1:TNH+CSu8EmXfitntjUPwaKVPN0AYMbc9F1bBS8/ABpw= +k8s.io/client-go v0.33.4/go.mod h1:LsA0+hBG2DPwovjd931L/AoaezMPX9CmBgyVyBZmbCY= +k8s.io/component-base v0.33.4 h1:Jvb/aw/tl3pfgnJ0E0qPuYLT0NwdYs1VXXYQmSuxJGY= +k8s.io/component-base v0.33.4/go.mod h1:567TeSdixWW2Xb1yYUQ7qk5Docp2kNznKL87eygY8Rc= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-aggregator v0.33.4 h1:TdIJKHb0/bLpby7FblXIaVEzyA1jGEjzt/n9cRvwq8U= +k8s.io/kube-aggregator v0.33.4/go.mod h1:wZuctdRvGde5bwzxkZRs0GYj2KOpCNgx8rRGVoNb62k= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= diff --git a/hack/deploy.sh b/hack/deploy.sh index ec4a4a9..8024978 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -151,23 +151,25 @@ else make docker-buildx IMG="${IMG}" fi -# --- 3. Secrets FIRST, so the manager mounts them on its very first boot ---- -# Ordering matters and lets us avoid any manager restart: -# - the webhook cert Secret is a REQUIRED volume mount, so it must exist -# before the pod starts; -# - provider credentials are read from the environment at process start. -# Both are consumed only at pod startup, so creating them before `make deploy` -# means the manager comes up already correct — no restart, no race. +# --- 3. Secrets FIRST, so the manager reads them on its very first boot ----- +# Provider credentials are read from the environment at process start, so creating +# them before `make deploy` means the manager comes up already correct — no restart, +# no race. (The webhook cert is NOT in this list: the manager mints it itself once +# running, see below.) # # Secrets need the namespace, which `make deploy` would create — so create it # up front (idempotent; kustomize re-applies it harmlessly during deploy). log "ensuring namespace ${NAMESPACE}" "${KUBECTL}" create namespace "${NAMESPACE}" --dry-run=client -o yaml | "${KUBECTL}" apply -f - -# Webhook serving cert (no cert-manager): generate the self-signed cert into the -# Secret now. The caBundle is injected later, after the webhook config exists. -log "provisioning webhook serving certificate Secret (self-signed)" -NAMESPACE="${NAMESPACE}" KUBECTL="${KUBECTL}" hack/gen-webhook-cert.sh secret +# No webhook cert step here, deliberately. The manager provisions its own serving +# cert in-process at startup (pkg/cert) — it mints the keypair into a Secret (which +# kustomize ships empty, and which the kubelet then projects into the pod for the +# webhook server to serve) and patches the caBundle into the +# MutatingWebhookConfiguration, then keeps RENEWING it before expiry. That last part is +# why it replaced the previous hack/gen-webhook-cert.sh: a script-minted cert never +# rotates, so its expiry is a time bomb that fires years later when nobody remembers +# the script exists. # Provider credential Secrets, one per provider (blank required keys → skipped). for row in "${PROVIDER_SECRETS[@]}"; do @@ -176,16 +178,20 @@ for row in "${PROVIDER_SECRETS[@]}"; do done # --- 4. install CRDs + deploy the manager ---------------------------------- -# The pod mounts the cert Secret and reads provider creds at boot — both already -# exist, so the manager comes up fully configured with no restart needed. +# The pod reads provider creds at boot and they already exist, so the manager comes +# up fully configured with no restart needed. log "installing CRDs and deploying the manager" make deploy IMG="${IMG}" -# --- 5. inject the webhook CA bundle (server-side, no manager restart) ------ -# This edits only the MutatingWebhookConfiguration, which just got created by -# `make deploy`. Only the API server reads caBundle, so the manager is untouched. -log "injecting webhook CA bundle" -NAMESPACE="${NAMESPACE}" KUBECTL="${KUBECTL}" hack/gen-webhook-cert.sh cabundle +# No CA-bundle injection step either: the manager patches its own caBundle once it +# starts, from the same cert it just wrote — so the served cert and the trusted CA +# cannot drift, which two separate steps could never fully guarantee. +# +# One consequence worth knowing when watching a first install: the manager registers +# NO controllers or webhook until that cert is ready, so the first few seconds of logs +# show "waiting for the webhook certificate to be ready" and nothing reconciles yet. +# That ordering is deliberate — a Pod admitted while the webhook is untrusted would be +# scheduled by vanilla Kubernetes and silently bypass placement. log "done. Check status with:" printf ' %s -n %s get pods\n' "${KUBECTL}" "${NAMESPACE}" diff --git a/hack/gen-webhook-cert.sh b/hack/gen-webhook-cert.sh deleted file mode 100755 index f5902a4..0000000 --- a/hack/gen-webhook-cert.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# -# gen-webhook-cert.sh — provision the webhook serving cert WITHOUT cert-manager. -# -# The webhook needs two things cert-manager would otherwise automate: -# 1. a TLS Secret (webhook-server-cert) the manager mounts and serves from; -# 2. that cert's CA injected into the MutatingWebhookConfiguration caBundle, -# so the API server trusts the webhook when it calls it. -# -# These two steps have DIFFERENT ordering requirements, so this script exposes -# them as separate actions (see usage) and a running manager never needs a -# restart: -# - the Secret must exist BEFORE the manager pod starts (it is a required -# volume mount), so create it before `make deploy`; -# - the caBundle patch is server-side (only the API server reads it), so it -# runs AFTER `make deploy` creates the webhook config. The manager is -# untouched by it. -# -# The caBundle is always derived from the cert already in the Secret, so the -# served cert and the trusted CA can never drift, even across re-runs. -# -# Usage: -# hack/gen-webhook-cert.sh secret # ensure the TLS Secret exists (generate if absent) -# hack/gen-webhook-cert.sh cabundle # inject the Secret's cert into the webhook config -# hack/gen-webhook-cert.sh all # both, in order (default; standalone use) -# -# Config (env / make flags): -# NAMESPACE namespace the manager runs in (default nebula-system) -# SERVICE webhook Service name (default nebula-webhook-service) -# SECRET TLS Secret the manager mounts (default webhook-server-cert) -# WEBHOOK_CONFIG MutatingWebhookConfiguration name (default nebula-mutating-webhook-configuration) -# CERT_DAYS certificate validity in days (default 3650) -# FORCE_REGEN if "true", regenerate even if the Secret exists (default false) -# KUBECTL kubectl binary (default kubectl) -# -# NOTE: the self-signed cert does NOT auto-rotate (that is cert-manager's main -# advantage). It is valid CERT_DAYS days; re-run with FORCE_REGEN=true to renew. -set -euo pipefail - -NAMESPACE="${NAMESPACE:-nebula-system}" -SERVICE="${SERVICE:-nebula-webhook-service}" -SECRET="${SECRET:-webhook-server-cert}" -WEBHOOK_CONFIG="${WEBHOOK_CONFIG:-nebula-mutating-webhook-configuration}" -CERT_DAYS="${CERT_DAYS:-3650}" -FORCE_REGEN="${FORCE_REGEN:-false}" -KUBECTL="${KUBECTL:-kubectl}" -ACTION="${1:-all}" - -log() { printf '\033[36m==>\033[0m %s\n' "$*"; } -die() { printf '\033[31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } - -command -v "${KUBECTL}" >/dev/null 2>&1 || die "kubectl not found on PATH" - -CN="${SERVICE}.${NAMESPACE}.svc" - -# ensure_secret — create the TLS Secret if absent (or if FORCE_REGEN=true). -# Leaves an existing Secret untouched so re-runs don't needlessly rotate the -# cert (which would otherwise require remounting on the manager). -ensure_secret() { - if [[ "${FORCE_REGEN}" != "true" ]] && \ - "${KUBECTL}" get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then - log "Secret ${SECRET} already exists in ${NAMESPACE}; keeping it (FORCE_REGEN=true to rotate)" - return 0 - fi - - command -v openssl >/dev/null 2>&1 || die "openssl not found on PATH" - - local tmp - tmp="$(mktemp -d)" - # shellcheck disable=SC2064 - trap "rm -rf '${tmp}'" RETURN - - log "generating self-signed cert for ${CN} (valid ${CERT_DAYS}d)" - # The cert is its own CA: it both serves TLS and is trusted via caBundle. - # SANs cover both DNS forms the webhook may be addressed by. - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "${tmp}/tls.key" -out "${tmp}/tls.crt" \ - -days "${CERT_DAYS}" -subj "/CN=${CN}" \ - -addext "subjectAltName=DNS:${CN},DNS:${CN}.cluster.local" >/dev/null 2>&1 - - log "applying TLS Secret ${SECRET} in ${NAMESPACE}" - "${KUBECTL}" create secret tls "${SECRET}" \ - --cert="${tmp}/tls.crt" --key="${tmp}/tls.key" \ - --namespace "${NAMESPACE}" \ - --dry-run=client -o yaml | "${KUBECTL}" apply -f - -} - -# inject_cabundle — read tls.crt from the Secret and set it as the webhook's -# caBundle. Deriving from the Secret guarantees the trusted CA matches the -# served cert. Requires the MutatingWebhookConfiguration to already exist. -inject_cabundle() { - "${KUBECTL}" get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1 \ - || die "Secret ${SECRET} not found in ${NAMESPACE}; run '$0 secret' first" - "${KUBECTL}" get mutatingwebhookconfiguration "${WEBHOOK_CONFIG}" >/dev/null 2>&1 \ - || die "${WEBHOOK_CONFIG} not found; deploy the manager (make deploy) before injecting the CA" - - # tls.crt in the Secret is already base64-encoded, which is exactly the form - # caBundle wants — no re-encoding needed. - local ca_b64 - ca_b64="$("${KUBECTL}" get secret "${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.crt}')" - [[ -n "${ca_b64}" ]] || die "Secret ${SECRET} has no tls.crt" - - log "injecting CA bundle into ${WEBHOOK_CONFIG}" - # JSON Patch "add" on an existing member replaces it, so this is correct on - # both first run (caBundle absent) and re-runs (caBundle present). - "${KUBECTL}" patch mutatingwebhookconfiguration "${WEBHOOK_CONFIG}" \ - --type=json \ - -p="[{\"op\":\"add\",\"path\":\"/webhooks/0/clientConfig/caBundle\",\"value\":\"${ca_b64}\"}]" -} - -case "${ACTION}" in - secret) ensure_secret ;; - cabundle) inject_cabundle ;; - all) ensure_secret; inject_cabundle ;; - *) die "unknown action '${ACTION}' (want: secret | cabundle | all)" ;; -esac diff --git a/internal/controller/pod_placement_controller.go b/internal/controller/pod_placement_controller.go index c201a4a..6ca65ca 100644 --- a/internal/controller/pod_placement_controller.go +++ b/internal/controller/pod_placement_controller.go @@ -199,7 +199,7 @@ func needsPlacement(pod *corev1.Pod) bool { if !pod.DeletionTimestamp.IsZero() { return false } - if pod.Labels[nebulav1alpha1.EnabledLabel] != "true" { + if pod.Labels[nebulav1alpha1.EnabledLabel] != nebulav1alpha1.EnabledValue { return false } if pod.Spec.NodeName != "" { diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index 0077795..a12c823 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -333,7 +333,7 @@ func removeGate(gates []corev1.PodSchedulingGate, name string) []corev1.PodSched // Delete is UID-pinned so a Pod already replaced by a same-name recreate is not // clobbered, and a NotFound (already gone) is treated as success. func (r *PodPlacementReconciler) reapTerminalPod(ctx context.Context, pod *corev1.Pod) (bool, error) { - if pod.Labels[nebulav1alpha1.EnabledLabel] != "true" { + if pod.Labels[nebulav1alpha1.EnabledLabel] != nebulav1alpha1.EnabledValue { return false, nil } if !pod.DeletionTimestamp.IsZero() { diff --git a/internal/controller/sandbox_controller.go b/internal/controller/sandbox_controller.go new file mode 100644 index 0000000..4d2982e --- /dev/null +++ b/internal/controller/sandbox_controller.go @@ -0,0 +1,437 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// sandboxContainerName is the name of the single container in a sandbox's Pod. +// It is fixed rather than user-settable because it is what `kubectl exec` and +// `kubectl logs` default to when no -c is given: a predictable name is the +// difference between `kubectl exec sbx-alice -- bash` working and the user having +// to look up a container name first. +const sandboxContainerName = "sandbox" + +// SandboxReconciler reconciles a Sandbox: it synthesizes the one Pod that backs +// the box, projects that Pod's status back onto the Sandbox, and enforces TTL. +// +// It deliberately does NOT talk to any provider. The Pod is the carrier for +// everything already built — the provider-selection gate, placement, the +// NodeClaim teardown ledger with its finalizer, ResourceQuota accounting — so +// this controller's whole job is to produce a correctly-shaped Pod and get out of +// the way. That is also why a Sandbox is not a bespoke provisioning path: bypassing +// the Pod would mean reimplementing the guarantee that a paid GPU is never leaked. +type SandboxReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxes/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete + +// Reconcile drives one Sandbox: ensure its Pod exists (unless the box is done), +// mirror the Pod's state into status, and release the instance when TTL elapses. +func (r *SandboxReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var sbx nebulav1alpha1.Sandbox + if err := r.Get(ctx, req.NamespacedName, &sbx); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !sbx.DeletionTimestamp.IsZero() { + // The Pod is ownerRef'd, so garbage collection deletes it, which triggers the + // virtual kubelet's teardown and the NodeClaim finalizer behind it. There is + // nothing for us to clean up, so we hold no finalizer of our own — one here + // would only add a way for the box to get stuck undeletable. + return ctrl.Result{}, nil + } + + // A terminal Sandbox keeps no instance. Its Pod has already been released, and + // recreating one would silently hand the user a DIFFERENT box (empty filesystem, + // new endpoint) under the same name, so we stop here and leave the object as the + // record of what happened. + if isTerminalSandboxPhase(sbx.Status.Phase) { + return ctrl.Result{}, nil + } + + pod, err := r.ensurePod(ctx, &sbx) + if err != nil { + return ctrl.Result{}, err + } + if pod == nil { + // A foreign Pod holds the name we need. Surface it and stop; this needs a human + // (rename the sandbox, or remove the squatter) and retrying cannot fix it. + log.Info("a Pod of this name exists but is not owned by this Sandbox", + "sandbox", sbx.Name, "pod", sbx.Name) + return ctrl.Result{}, r.setStatus(ctx, &sbx, sandboxStatus{ + phase: nebulav1alpha1.SandboxPending, + reason: nebulav1alpha1.ReasonPodConflict, + msg: "a Pod named " + sbx.Name + " already exists and is not owned by this Sandbox", + }) + } + + // TTL is measured from the moment the box became READY, not from creation, so a + // slow provision does not eat into the user's time. That means the deadline only + // exists once ReadyTime is set, and expiry is checked before status is refreshed + // so an expiring box reports Expired rather than briefly re-reporting Ready. + if expired, err := r.enforceTTL(ctx, &sbx, pod); err != nil || expired { + return ctrl.Result{}, err + } + + st := sandboxStatusFromPod(pod) + if err := r.setStatus(ctx, &sbx, st); err != nil { + return ctrl.Result{}, err + } + + // Requeue for the exact moment TTL elapses. Nothing emits an event when a + // deadline passes, so without this the box would bill until the next periodic + // resync — the failure mode TTL exists to prevent. + if until, ok := timeUntilExpiry(&sbx); ok { + return ctrl.Result{RequeueAfter: until}, nil + } + return ctrl.Result{}, nil +} + +// ensurePod creates the Sandbox's Pod if absent and returns it. It returns +// (nil, nil) when a Pod of the required name exists but is NOT owned by this +// Sandbox: adoption is refused because the Pod could be an unrelated workload, +// and adopting it would subject someone else's Pod to this Sandbox's lifecycle +// (including deletion on TTL expiry). The caller surfaces that as a condition. +func (r *SandboxReconciler) ensurePod(ctx context.Context, sbx *nebulav1alpha1.Sandbox) (*corev1.Pod, error) { + var existing corev1.Pod + err := r.Get(ctx, client.ObjectKey{Namespace: sbx.Namespace, Name: sbx.Name}, &existing) + if err == nil { + if !isOwnedBy(&existing, sbx) { + return nil, nil + } + return &existing, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + + pod := r.buildPod(sbx) + if err := controllerutil.SetControllerReference(sbx, pod, r.Scheme); err != nil { + return nil, err + } + if err := r.Create(ctx, pod); err != nil { + if apierrors.IsAlreadyExists(err) { + // Lost a race with another reconcile (or with a foreign creator). Re-read on + // the next pass rather than guessing which it was. + return nil, nil + } + return nil, err + } + return pod, nil +} + +// buildPod synthesizes the Pod that backs the sandbox. Everything Nebula's +// existing placement path requires is stamped here, so a Sandbox's Pod is +// indistinguishable from a correctly hand-written one — the same webhook gates it, +// the same placement controller places it, the same virtual kubelet provisions it. +func (r *SandboxReconciler) buildPod(sbx *nebulav1alpha1.Sandbox) *corev1.Pod { + labels := map[string]string{ + // EnabledLabel is the opt-in the mutating webhook selects on: without it the + // Pod would be scheduled by vanilla Kubernetes and never reach a provider. + nebulav1alpha1.EnabledLabel: nebulav1alpha1.EnabledValue, + nebulav1alpha1.ManagedByLabel: nebulav1alpha1.ManagedByValue, + nebulav1alpha1.PoolLabel: sbx.Spec.NodePoolRef, + nebulav1alpha1.SandboxLabel: sbx.Name, + } + if sbx.Spec.AcceleratorType != "" { + // The TYPE is a label and the COUNT is an nvidia.com/gpu resource — the split + // the rest of the system already reads (see util.AcceleratorRequest). A + // CPU-only sandbox sets neither. + labels[nebulav1alpha1.AcceleratorTypeLabel] = sbx.Spec.AcceleratorType + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + // Same name as the Sandbox, so `kubectl exec sbx-alice` and + // `kubectl logs sbx-alice` work with the name the user already knows. The + // name is still published in status.PodName so tooling reads it from there + // rather than depending on this being true forever. + Name: sbx.Name, + Namespace: sbx.Namespace, + Labels: labels, + }, + Spec: corev1.PodSpec{ + // Never restart: the provider owns the instance lifecycle, and a sandbox + // whose instance is gone must surface as Failed rather than being silently + // replaced by an empty box wearing the same name. + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: sandboxContainerName, + Image: sbx.Spec.Image, + // SandD is the command, set HERE rather than in each provider's bootstrap, + // so the Pod stays the single source of truth for what runs on the instance + // — the same rule ProvisionRequest documents for image/env/resources. Every + // adapter already reads the command off the Pod, so this needs no + // per-provider code and cannot drift between providers. + // + // SandD is PID 1 in the container. For a sandbox it spawns nothing: the box + // exists to receive exec calls, so SandD just holds the container open and + // serves them. For workload classes that DO run something, it spawns that as + // its child and owns the stdout/stderr pipes, which is what makes + // `kubectl logs` work against an instance in another cloud. + // + // The user's image does not contain this binary (it is an arbitrary image + // like ubuntu:24.04), so the provider bootstrap must make it appear at + // SanddPath — the contract the shared constant exists to pin down. This is + // never in tension with a user-supplied command: SandboxSpec has no command + // field, so the structural schema rejects one outright. + Command: []string{nebulav1alpha1.SanddPath}, + Resources: sbx.Spec.Resources, + Env: sbx.Spec.Env, + }}, + }, + } +} + +// sandboxStatus is the projection this controller writes, gathered in one struct +// so status is set through a single path (and thus a single Status().Update). +type sandboxStatus struct { + phase nebulav1alpha1.SandboxPhase + reason string + msg string + endpoint string + // ready marks the Ready condition True. It is separate from phase because only + // SandboxReady implies readiness, and conditions and phases are updated together. + ready bool +} + +// sandboxStatusFromPod projects the backing Pod's state onto the Sandbox. The Pod +// (via the virtual kubelet) is the source of truth for what the external instance +// is doing, so this reads it rather than tracking instance state independently — +// two sources for one fact is how they drift. +// +// It deliberately takes only the Pod, not the Sandbox: the projection must depend +// on nothing but observed Pod state, or a stale value already on Sandbox.Status +// could feed back into the next projection and latch. +// +// The mapping keys off the Pod's status REASON, not just its phase, because the +// interesting distinction for a user is inside PodPending: "cannot get capacity" +// (Provisioning) versus "capacity granted, still booting" (Initializing). The +// vnode stamps those reasons (see pkg/vnode/status.go). +func sandboxStatusFromPod(pod *corev1.Pod) sandboxStatus { + endpoint := pod.Annotations[nebulav1alpha1.EndpointAnnotation] + + switch pod.Status.Phase { + case corev1.PodRunning: + if isPodReady(pod) { + return sandboxStatus{ + phase: nebulav1alpha1.SandboxReady, + reason: nebulav1alpha1.ReasonSandboxReady, + msg: "the sandbox instance is running and reachable", + endpoint: endpoint, + ready: true, + } + } + return sandboxStatus{ + phase: nebulav1alpha1.SandboxInitializing, + reason: nebulav1alpha1.ReasonSandboxProvisioning, + msg: "the sandbox instance is running but not yet ready", + endpoint: endpoint, + } + case corev1.PodFailed, corev1.PodSucceeded: + // Succeeded lands here too: a sandbox has no notion of completing — SandD is + // PID 1 and only exits when the box goes away — so a terminal Pod means the + // instance is gone either way. + return sandboxStatus{ + phase: nebulav1alpha1.SandboxFailed, + reason: nebulav1alpha1.ReasonSandboxFailed, + msg: podFailureMessage(pod), + endpoint: endpoint, + } + default: + // Pending. A Pod still held by the provider-selection gate has not been placed + // at all, which is a different problem from a provision in flight: it means no + // provider in the pool can serve this box right now. Distinguishing them is + // what stops a capacity problem from looking like a slow boot. + if hasGateNamed(pod) { + return sandboxStatus{ + phase: nebulav1alpha1.SandboxPending, + reason: nebulav1alpha1.ReasonSandboxProvisioning, + msg: "waiting for placement onto a provider", + } + } + phase := nebulav1alpha1.SandboxProvisioning + if pod.Status.Reason == podReasonInitializing { + phase = nebulav1alpha1.SandboxInitializing + } + return sandboxStatus{ + phase: phase, + reason: nebulav1alpha1.ReasonSandboxProvisioning, + msg: podStatusMessage(pod, "bringing the sandbox instance up"), + endpoint: endpoint, + } + } +} + +// setStatus writes the projection, stamping ReadyTime/ExpiryTime on the first +// transition to Ready. It skips the API call when nothing changed, so a Sandbox +// that is simply sitting Ready does not generate an update per resync. +func (r *SandboxReconciler) setStatus(ctx context.Context, sbx *nebulav1alpha1.Sandbox, st sandboxStatus) error { + before := sbx.Status.DeepCopy() + + sbx.Status.Phase = st.phase + sbx.Status.PodName = sbx.Name + if st.endpoint != "" { + sbx.Status.Endpoint = st.endpoint + } + + // ReadyTime is durable and written exactly once: it anchors TTL, so recomputing + // it from the Pod would let a status blip silently restart the user's clock. + if st.phase == nebulav1alpha1.SandboxReady && sbx.Status.ReadyTime == nil { + now := metav1.Now() + sbx.Status.ReadyTime = &now + if ttl := sbx.Spec.TTL; ttl != nil && ttl.Duration > 0 { + expiry := metav1.NewTime(now.Add(ttl.Duration)) + sbx.Status.ExpiryTime = &expiry + } + } + + condStatus := metav1.ConditionFalse + if st.ready { + condStatus = metav1.ConditionTrue + } + apimeta.SetStatusCondition(&sbx.Status.Conditions, metav1.Condition{ + Type: nebulav1alpha1.SandboxConditionReady, + Status: condStatus, + Reason: st.reason, + Message: st.msg, + ObservedGeneration: sbx.Generation, + }) + + // Skip the write when only the condition's LastTransitionTime would differ — + // SetStatusCondition preserves it when Status is unchanged, so a semantic compare + // is enough to keep a steady-state Ready sandbox from generating an update per + // resync (each of which would wake every watcher). + if apiequality.Semantic.DeepEqual(before, &sbx.Status) { + return nil + } + return r.Status().Update(ctx, sbx) +} + +// enforceTTL releases the instance when the box's deadline has passed, by +// deleting the backing Pod — which is what triggers the virtual kubelet's +// teardown and, behind it, the NodeClaim finalizer that guarantees the paid +// instance is actually reclaimed. Deleting the Pod (rather than the Sandbox) +// leaves the object as the record of why the box went away, so a user returning to +// an expired sandbox gets an answer instead of a NotFound. +// +// It returns expired=true when it acted, so the caller stops reconciling this pass. +func (r *SandboxReconciler) enforceTTL(ctx context.Context, sbx *nebulav1alpha1.Sandbox, pod *corev1.Pod) (bool, error) { + if sbx.Status.ExpiryTime == nil || time.Now().Before(sbx.Status.ExpiryTime.Time) { + return false, nil + } + + // UID-pinned so a Pod already replaced by something else is never clobbered; + // an already-gone Pod is success, not an error. + preconditions := metav1.Preconditions{UID: &pod.UID} + if err := r.Delete(ctx, pod, &client.DeleteOptions{Preconditions: &preconditions}); err != nil { + if !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return false, err + } + } + return true, r.setStatus(ctx, sbx, sandboxStatus{ + phase: nebulav1alpha1.SandboxExpired, + reason: nebulav1alpha1.ReasonSandboxExpired, + msg: "ttl elapsed; the sandbox instance was released", + }) +} + +// timeUntilExpiry reports how long until the box's TTL elapses, and whether +// there is a deadline at all. A deadline already in the past returns a small +// positive delay rather than zero, because a zero RequeueAfter means "do not +// requeue" to controller-runtime — precisely the wrong reading for an overdue box. +func timeUntilExpiry(sbx *nebulav1alpha1.Sandbox) (time.Duration, bool) { + if sbx.Status.ExpiryTime == nil { + return 0, false + } + until := time.Until(sbx.Status.ExpiryTime.Time) + if until <= 0 { + return time.Second, true + } + return until, true +} + +// isTerminalSandboxPhase reports whether the box is done and holds no instance. +func isTerminalSandboxPhase(p nebulav1alpha1.SandboxPhase) bool { + return p == nebulav1alpha1.SandboxExpired || p == nebulav1alpha1.SandboxFailed +} + +// isOwnedBy reports whether obj is controlled by the given Sandbox, matching on +// UID so a recreated Sandbox of the same name does not adopt the old box's Pod. +func isOwnedBy(obj client.Object, sbx *nebulav1alpha1.Sandbox) bool { + ref := metav1.GetControllerOf(obj) + return ref != nil && ref.UID == sbx.UID +} + +// isPodReady reports whether the Pod's Ready condition is True. The virtual +// kubelet sets it when the provider reports the instance running and reachable. +func isPodReady(pod *corev1.Pod) bool { + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +// podFailureMessage explains why the box died, preferring the Pod's own message +// (the vnode writes a specific one: provision rejected, instance gone, instance +// failed) over a generic fallback. +func podFailureMessage(pod *corev1.Pod) string { + return podStatusMessage(pod, "the sandbox instance is no longer running") +} + +// podStatusMessage returns the Pod's status message, or fallback when it has none. +func podStatusMessage(pod *corev1.Pod, fallback string) string { + if pod.Status.Message != "" { + return pod.Status.Message + } + return fallback +} + +// SetupWithManager wires the controller. It owns its Pod, so a Pod status change +// (the instance coming up, failing, or vanishing) re-reconciles the Sandbox +// immediately instead of waiting for the periodic resync. +func (r *SandboxReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nebulav1alpha1.Sandbox{}). + Owns(&corev1.Pod{}). + Named("sandbox"). + Complete(r) +} diff --git a/internal/controller/sandbox_controller_test.go b/internal/controller/sandbox_controller_test.go new file mode 100644 index 0000000..915c0f1 --- /dev/null +++ b/internal/controller/sandbox_controller_test.go @@ -0,0 +1,454 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/util" +) + +const ( + testNS = "team-ml" + testSbxName = "alice" +) + +// newSandboxReconciler builds a reconciler over a fake client seeded with objs. +func newSandboxReconciler(objs ...client.Object) (*SandboxReconciler, client.Client) { + s := runtime.NewScheme() + _ = clientgoscheme.AddToScheme(s) + _ = nebulav1alpha1.AddToScheme(s) + c := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&nebulav1alpha1.Sandbox{}). + Build() + return &SandboxReconciler{Client: c, Scheme: s}, c +} + +// newSandbox is a Sandbox with the fields the controller reads. The UID is set +// because ownership checks compare it, and the fake client does not assign one. +func newSandbox(mutators ...func(*nebulav1alpha1.Sandbox)) *nebulav1alpha1.Sandbox { + sbx := &nebulav1alpha1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSbxName, + Namespace: testNS, + UID: types.UID("sbx-uid-1"), + }, + Spec: nebulav1alpha1.SandboxSpec{ + NodePoolRef: "gpu", + Image: "ubuntu:24.04", + AcceleratorType: "a100-40gb", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + util.NvidiaGPUResource: resource.MustParse("1"), + }, + }, + }, + } + for _, m := range mutators { + m(sbx) + } + return sbx +} + +func reconcileSandbox(t *testing.T, r *SandboxReconciler) reconcile.Result { + t.Helper() + res, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: testNS, Name: testSbxName}, + }) + if err != nil { + t.Fatalf("Reconcile: %v", err) + } + return res +} + +func getSandboxPod(t *testing.T, c client.Client) *corev1.Pod { + t.Helper() + var pod corev1.Pod + key := client.ObjectKey{Namespace: testNS, Name: testSbxName} + if err := c.Get(context.Background(), key, &pod); err != nil { + t.Fatalf("get pod: %v", err) + } + return &pod +} + +func getSandbox(t *testing.T, c client.Client) *nebulav1alpha1.Sandbox { + t.Helper() + var sbx nebulav1alpha1.Sandbox + key := client.ObjectKey{Namespace: testNS, Name: testSbxName} + if err := c.Get(context.Background(), key, &sbx); err != nil { + t.Fatalf("get sandbox: %v", err) + } + return &sbx +} + +// TestSandboxSynthesizesPod covers the whole contract the synthesized Pod must +// satisfy for the EXISTING placement path to pick it up unchanged. Each assertion +// here is load-bearing: drop the opt-in label and the Pod is scheduled by vanilla +// Kubernetes and never reaches a provider; drop the pool label and placement has +// no policy to resolve. +func TestSandboxSynthesizesPod(t *testing.T) { + r, c := newSandboxReconciler(newSandbox()) + reconcileSandbox(t, r) + pod := getSandboxPod(t, c) + + if got := pod.Labels[nebulav1alpha1.EnabledLabel]; got != "true" { + t.Errorf("opt-in label = %q, want \"true\" (without it the Pod never reaches a provider)", got) + } + if got := pod.Labels[nebulav1alpha1.PoolLabel]; got != "gpu" { + t.Errorf("pool label = %q, want \"gpu\"", got) + } + if got := pod.Labels[nebulav1alpha1.SandboxLabel]; got != testSbxName { + t.Errorf("sandbox label = %q, want %q", got, testSbxName) + } + if got := pod.Labels[nebulav1alpha1.AcceleratorTypeLabel]; got != "a100-40gb" { + t.Errorf("accelerator label = %q, want \"a100-40gb\"", got) + } + if pod.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Errorf("restartPolicy = %q, want Never", pod.Spec.RestartPolicy) + } + + if n := len(pod.Spec.Containers); n != 1 { + t.Fatalf("containers = %d, want 1", n) + } + ctr := pod.Spec.Containers[0] + if ctr.Name != sandboxContainerName { + t.Errorf("container name = %q, want %q (kubectl exec defaults to it)", ctr.Name, sandboxContainerName) + } + if ctr.Image != "ubuntu:24.04" { + t.Errorf("image = %q, want ubuntu:24.04", ctr.Image) + } + // SandD must be the command: it is PID 1 and serves exec/logs. A Pod without it + // would run the image's own entrypoint and be unreachable. + if len(ctr.Command) != 1 || ctr.Command[0] != nebulav1alpha1.SanddPath { + t.Errorf("command = %v, want [%s]", ctr.Command, nebulav1alpha1.SanddPath) + } + // The GPU count must survive as a standard resource: placement and the + // scheduler's fit check both read it from here. + if q, ok := ctr.Resources.Limits[util.NvidiaGPUResource]; !ok || q.Value() != 1 { + t.Errorf("nvidia.com/gpu limit = %v (present=%v), want 1", q.Value(), ok) + } + + // Controller-owned, so garbage collection releases the instance when the + // Sandbox is deleted. + ref := metav1.GetControllerOf(pod) + if ref == nil || ref.Kind != "Sandbox" || ref.Name != testSbxName { + t.Errorf("controller ref = %+v, want the Sandbox", ref) + } +} + +// TestSandboxCPUOnlyOmitsAcceleratorLabel: a CPU-only box must not carry an empty +// accelerator label, which would make placement look for an accelerator named "". +func TestSandboxCPUOnlyOmitsAcceleratorLabel(t *testing.T) { + sbx := newSandbox(func(s *nebulav1alpha1.Sandbox) { + s.Spec.AcceleratorType = "" + s.Spec.Resources = corev1.ResourceRequirements{} + }) + r, c := newSandboxReconciler(sbx) + reconcileSandbox(t, r) + + if _, present := getSandboxPod(t, c).Labels[nebulav1alpha1.AcceleratorTypeLabel]; present { + t.Error("CPU-only sandbox must not set the accelerator-type label") + } +} + +// TestSandboxIsIdempotent: a second reconcile must not create a second Pod nor +// error. Controllers are re-run constantly, so this is the baseline invariant. +func TestSandboxIsIdempotent(t *testing.T) { + r, c := newSandboxReconciler(newSandbox()) + reconcileSandbox(t, r) + reconcileSandbox(t, r) + + var pods corev1.PodList + if err := c.List(context.Background(), &pods, client.InNamespace(testNS)); err != nil { + t.Fatalf("list pods: %v", err) + } + if len(pods.Items) != 1 { + t.Fatalf("pods = %d, want 1 (reconcile must be idempotent)", len(pods.Items)) + } +} + +// TestSandboxPhaseFromPod checks the projection from Pod state to Sandbox phase. +// The distinction that matters most is inside PodPending: a gated Pod means "no +// provider can serve this box" while an ungated one means "provisioning is under +// way" — conflating them would make a capacity problem look like a slow boot. +func TestSandboxPhaseFromPod(t *testing.T) { + gated := func(p *corev1.Pod) { + p.Spec.SchedulingGates = []corev1.PodSchedulingGate{ + {Name: nebulav1alpha1.ProviderSelectionGate}, + } + } + tests := []struct { + name string + mutate func(*corev1.Pod) + want nebulav1alpha1.SandboxPhase + wantRdy bool + }{ + { + name: "gated pod is Pending, not Provisioning", + mutate: func(p *corev1.Pod) { gated(p); p.Status.Phase = corev1.PodPending }, + want: nebulav1alpha1.SandboxPending, + }, + { + name: "ungated pending pod is Provisioning", + mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodPending }, + want: nebulav1alpha1.SandboxProvisioning, + }, + { + name: "pending with Initializing reason is Initializing", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodPending + p.Status.Reason = podReasonInitializing + }, + want: nebulav1alpha1.SandboxInitializing, + }, + { + name: "running but not ready is Initializing", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodRunning + }, + want: nebulav1alpha1.SandboxInitializing, + }, + { + name: "running and ready is Ready", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodRunning + p.Status.Conditions = []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + } + }, + want: nebulav1alpha1.SandboxReady, + wantRdy: true, + }, + { + name: "failed pod is Failed", + mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodFailed }, + want: nebulav1alpha1.SandboxFailed, + }, + { + // SandD only exits when the box goes away, so a Succeeded Pod still means + // the instance is gone — not that the sandbox completed successfully. + name: "succeeded pod is Failed too", + mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodSucceeded }, + want: nebulav1alpha1.SandboxFailed, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + sbx := newSandbox() + r, c := newSandboxReconciler(sbx) + reconcileSandbox(t, r) // creates the Pod + + pod := getSandboxPod(t, c) + tc.mutate(pod) + // Spec and status must be written separately, and the status has to be + // re-applied after the spec write: the fake client treats Pod status as a + // subresource, so Update copies the STORED status back over the object it + // was handed, discarding what mutate just set. + want := pod.Status + if err := c.Update(context.Background(), pod); err != nil { + t.Fatalf("update pod: %v", err) + } + pod.Status = want + if err := c.Status().Update(context.Background(), pod); err != nil { + t.Fatalf("update pod status: %v", err) + } + + reconcileSandbox(t, r) + got := getSandbox(t, c) + if got.Status.Phase != tc.want { + t.Errorf("phase = %q, want %q", got.Status.Phase, tc.want) + } + ready := false + for _, cond := range got.Status.Conditions { + if cond.Type == nebulav1alpha1.SandboxConditionReady { + ready = cond.Status == metav1.ConditionTrue + } + } + if ready != tc.wantRdy { + t.Errorf("Ready condition = %v, want %v", ready, tc.wantRdy) + } + }) + } +} + +// TestSandboxEndpointAndReadyTime: the endpoint must be mirrored from the Pod +// annotation (it is the only way to reach the box), and ReadyTime must be stamped +// once so TTL has a stable anchor. +func TestSandboxEndpointAndReadyTime(t *testing.T) { + r, c := newSandboxReconciler(newSandbox(func(s *nebulav1alpha1.Sandbox) { + s.Spec.TTL = &metav1.Duration{Duration: time.Hour} + })) + reconcileSandbox(t, r) + markPodReady(t, c, "ec2-1-2-3-4.compute.amazonaws.com") + reconcileSandbox(t, r) + + sbx := getSandbox(t, c) + if sbx.Status.Endpoint != "ec2-1-2-3-4.compute.amazonaws.com" { + t.Errorf("endpoint = %q, want the Pod's annotation value", sbx.Status.Endpoint) + } + if sbx.Status.ReadyTime == nil { + t.Fatal("ReadyTime must be stamped on the first transition to Ready") + } + if sbx.Status.ExpiryTime == nil { + t.Fatal("ExpiryTime must be derived from ReadyTime + TTL") + } + firstReady := sbx.Status.ReadyTime.DeepCopy() + + // A later reconcile must NOT move ReadyTime: it anchors TTL, so re-deriving it + // would let a status blip silently restart the user's clock. + reconcileSandbox(t, r) + if got := getSandbox(t, c).Status.ReadyTime; !got.Equal(firstReady) { + t.Errorf("ReadyTime moved from %v to %v; it must be written exactly once", firstReady, got) + } +} + +// TestSandboxTTLReleasesInstance: once the deadline passes the Pod must be +// deleted — that is what triggers VK teardown and the NodeClaim finalizer behind +// it — while the Sandbox object survives as the record of why the box went away. +func TestSandboxTTLReleasesInstance(t *testing.T) { + r, c := newSandboxReconciler(newSandbox(func(s *nebulav1alpha1.Sandbox) { + s.Spec.TTL = &metav1.Duration{Duration: time.Hour} + })) + reconcileSandbox(t, r) + markPodReady(t, c, "1.2.3.4") + reconcileSandbox(t, r) + + // Backdate the expiry rather than sleeping. + sbx := getSandbox(t, c) + past := metav1.NewTime(time.Now().Add(-time.Minute)) + sbx.Status.ExpiryTime = &past + if err := c.Status().Update(context.Background(), sbx); err != nil { + t.Fatalf("update status: %v", err) + } + + reconcileSandbox(t, r) + + var pod corev1.Pod + err := c.Get(context.Background(), client.ObjectKey{Namespace: testNS, Name: testSbxName}, &pod) + if !apierrors.IsNotFound(err) { + t.Errorf("Pod must be deleted on expiry so the instance is released; get err = %v", err) + } + if got := getSandbox(t, c).Status.Phase; got != nebulav1alpha1.SandboxExpired { + t.Errorf("phase = %q, want Expired", got) + } +} + +// TestSandboxTerminalDoesNotRecreatePod: an expired or failed box must stay dead. +// Recreating the Pod would hand the user a DIFFERENT box (empty filesystem, new +// endpoint) under the same name — the single most confusing thing this controller +// could do. +func TestSandboxTerminalDoesNotRecreatePod(t *testing.T) { + for _, phase := range []nebulav1alpha1.SandboxPhase{ + nebulav1alpha1.SandboxExpired, + nebulav1alpha1.SandboxFailed, + } { + t.Run(string(phase), func(t *testing.T) { + sbx := newSandbox() + sbx.Status.Phase = phase + r, c := newSandboxReconciler(sbx) + + reconcileSandbox(t, r) + + var pods corev1.PodList + if err := c.List(context.Background(), &pods, client.InNamespace(testNS)); err != nil { + t.Fatalf("list pods: %v", err) + } + if len(pods.Items) != 0 { + t.Errorf("pods = %d, want 0: a terminal sandbox must not be resurrected", len(pods.Items)) + } + }) + } +} + +// TestSandboxRefusesForeignPod: a Pod of the required name that belongs to someone +// else must NOT be adopted — adopting would subject an unrelated workload to this +// Sandbox's lifecycle, including deletion on TTL expiry. +func TestSandboxRefusesForeignPod(t *testing.T) { + foreign := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSbxName, + Namespace: testNS, + Labels: map[string]string{"app": "someone-elses-thing"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "app", Image: "nginx"}}, + }, + } + r, c := newSandboxReconciler(newSandbox(), foreign) + reconcileSandbox(t, r) + + sbx := getSandbox(t, c) + if sbx.Status.Phase != nebulav1alpha1.SandboxPending { + t.Errorf("phase = %q, want Pending", sbx.Status.Phase) + } + var reason string + for _, cond := range sbx.Status.Conditions { + if cond.Type == nebulav1alpha1.SandboxConditionReady { + reason = cond.Reason + } + } + if reason != nebulav1alpha1.ReasonPodConflict { + t.Errorf("condition reason = %q, want %q", reason, nebulav1alpha1.ReasonPodConflict) + } + + // The foreign Pod must be untouched. + pod := getSandboxPod(t, c) + if pod.Labels["app"] != "someone-elses-thing" { + t.Error("the foreign Pod was mutated; it must be left alone") + } + if len(pod.Spec.Containers) != 1 || pod.Spec.Containers[0].Image != "nginx" { + t.Error("the foreign Pod's spec was overwritten") + } +} + +// markPodReady drives the Sandbox's Pod to Running+Ready with an endpoint, the way +// the virtual kubelet would once the provider reports the instance up. +func markPodReady(t *testing.T, c client.Client, endpoint string) { + t.Helper() + pod := getSandboxPod(t, c) + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[nebulav1alpha1.EndpointAnnotation] = endpoint + if err := c.Update(context.Background(), pod); err != nil { + t.Fatalf("update pod: %v", err) + } + pod.Status.Phase = corev1.PodRunning + pod.Status.Conditions = []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + } + if err := c.Status().Update(context.Background(), pod); err != nil { + t.Fatalf("update pod status: %v", err) + } +} diff --git a/internal/controller/sandbox_validation_test.go b/internal/controller/sandbox_validation_test.go new file mode 100644 index 0000000..8999dd0 --- /dev/null +++ b/internal/controller/sandbox_validation_test.go @@ -0,0 +1,174 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/util" +) + +// These specs exercise Sandbox admission — the CEL rule on SandboxSpec and the +// schema-level guarantees (image defaulting, and the absence of a command field). +// None of it is enforced by the fake client the unit tests use: it runs neither +// x-kubernetes-validations nor structural-schema defaulting/pruning, so a real +// apiserver is the only thing that can prove these hold. Requires envtest binaries; +// the whole suite is skipped in BeforeSuite when they are absent. +var _ = Describe("Sandbox admission", func() { + newSandbox := func(name string) *nebulav1alpha1.Sandbox { + return &nebulav1alpha1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: nebulav1alpha1.SandboxSpec{ + NodePoolRef: "sample", + }, + } + } + gpuLimit := func(sbx *nebulav1alpha1.Sandbox, n string) { + sbx.Spec.Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse(n)}, + } + } + + It("rejects a GPU count with no acceleratorType", func() { + // The contradictory pair: util.AcceleratorRequest errors on it, so admitting + // this would defer the failure to placement — minutes later, and reported on the + // Pod rather than on the object the user actually wrote. + sbx := newSandbox("gpu-no-type") + gpuLimit(sbx, "1") + + err := k8sClient.Create(ctx, sbx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nvidia.com/gpu requires acceleratorType to be set")) + }) + + It("rejects a GPU request (not just a limit) with no acceleratorType", func() { + // gpuCount reads limits OR requests, so the rule has to cover both; a + // requests-only spec would otherwise slip through and fail at placement. + sbx := newSandbox("gpu-request-no-type") + sbx.Spec.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse("1")}, + } + + err := k8sClient.Create(ctx, sbx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nvidia.com/gpu requires acceleratorType to be set")) + }) + + It("admits a GPU count together with an acceleratorType", func() { + sbx := newSandbox("gpu-with-type") + sbx.Spec.AcceleratorType = "a100-40gb" + gpuLimit(sbx, "1") + + Expect(k8sClient.Create(ctx, sbx)).To(Succeed()) + Expect(k8sClient.Delete(ctx, sbx)).To(Succeed()) + }) + + It("admits an acceleratorType with no count (which means one accelerator)", func() { + sbx := newSandbox("type-no-count") + sbx.Spec.AcceleratorType = "h100" + + Expect(k8sClient.Create(ctx, sbx)).To(Succeed()) + Expect(k8sClient.Delete(ctx, sbx)).To(Succeed()) + }) + + It("admits a CPU-only sandbox with non-GPU resources", func() { + sbx := newSandbox("cpu-only") + sbx.Spec.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("4")}, + } + + Expect(k8sClient.Create(ctx, sbx)).To(Succeed()) + Expect(k8sClient.Delete(ctx, sbx)).To(Succeed()) + }) + + It("defaults the image so a bare spec is usable", func() { + sbx := newSandbox("default-image") + Expect(k8sClient.Create(ctx, sbx)).To(Succeed()) + defer func() { Expect(k8sClient.Delete(ctx, sbx)).To(Succeed()) }() + + var got nebulav1alpha1.Sandbox + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(sbx), &got)).To(Succeed()) + Expect(got.Spec.Image).To(Equal("ubuntu:24.04")) + }) + + It("rejects an explicitly empty image rather than defaulting it", func() { + // `image: ""` is a mistake, not a request for the default, and MinLength must + // still bite — otherwise the box would boot something its author never named. + // + // This has to go through unstructured: the field is `omitempty`, so a Go zero + // value is dropped before the request is sent and would be DEFAULTED instead of + // rejected. Only an explicit empty string on the wire reaches MinLength, which is + // exactly the distinction the apiserver draws between unset and empty. + sbx := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nebulav1alpha1.GroupVersion.String(), + "kind": "Sandbox", + "metadata": map[string]any{"name": "empty-image", "namespace": "default"}, + "spec": map[string]any{ + "nodePoolRef": "sample", + "image": "", + }, + }} + + err := k8sClient.Create(ctx, sbx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("should be at least 1 chars long")) + }) + + It("rejects a spec with no nodePoolRef", func() { + // Placing a paid GPU instance against a guessed policy is not a safe default, + // so the field is required rather than defaulted. + sbx := newSandbox("no-pool") + sbx.Spec.NodePoolRef = "" + + err := k8sClient.Create(ctx, sbx) + Expect(err).To(HaveOccurred()) + }) + + It("rejects a user-supplied command", func() { + // The process model depends on SandD being PID 1 — it is what serves exec and + // logs — so a command must never reach the container. SandboxSpec simply has no + // command field, and because the CRD is a structural schema the apiserver rejects + // the unknown field itself. This spec exists because that guarantee is the reason + // no validating webhook was written: if the schema ever stopped rejecting it (a + // stray x-kubernetes-preserve-unknown-fields would do it), the command would be + // silently pruned instead, and the failure would surface as "exec does not work" + // rather than as a rejected object. + sbx := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nebulav1alpha1.GroupVersion.String(), + "kind": "Sandbox", + "metadata": map[string]any{"name": "with-command", "namespace": "default"}, + "spec": map[string]any{ + "nodePoolRef": "sample", + "command": []any{"/bin/sleep", "infinity"}, + }, + }} + + // Strict field validation is what turns "unknown field" from a silent prune into + // an error; kubectl applies it by default, so this matches what a user sees. + err := k8sClient.Create(ctx, sbx, client.FieldValidation(metav1.FieldValidationStrict)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unknown field")) + }) +}) diff --git a/internal/controller/sandboxset_controller.go b/internal/controller/sandboxset_controller.go new file mode 100644 index 0000000..4d7cf19 --- /dev/null +++ b/internal/controller/sandboxset_controller.go @@ -0,0 +1,354 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "sort" + + apiequality "k8s.io/apimachinery/pkg/api/equality" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// SandboxSetReconciler reconciles a SandboxSet: it creates and deletes Sandbox +// objects so that spec.Replicas of them exist, and rolls their readiness up into +// the set's status. +// +// It deliberately knows nothing about Pods, instances, or providers. Its entire +// vocabulary is Sandbox objects, and the Sandbox controller handles what one box +// means — which is what makes the two-type split pay off: scaling logic and box +// lifecycle never tangle. +// +// A template change does NOT roll existing boxes. This mirrors ReplicaSet, not +// Deployment, and here it is the only defensible behaviour: a rolling update would +// evict live sessions and burn minutes of provisioning per box to deliver a change +// nobody attached to a running sandbox asked for. New boxes get the new template; +// existing ones are left alone until something else removes them. +type SandboxSetReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxsets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxsets/finalizers,verbs=update +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxsets/scale,verbs=get;update;patch +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=sandboxes,verbs=get;list;watch;create;update;patch;delete + +// Reconcile brings the owned Sandbox count to spec.Replicas and refreshes status. +func (r *SandboxSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var set nebulav1alpha1.SandboxSet + if err := r.Get(ctx, req.NamespacedName, &set); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !set.DeletionTimestamp.IsZero() { + // Owned Sandboxes are ownerRef'd, so garbage collection removes them and each + // Sandbox controller releases its own instance. Nothing to do here, and no + // finalizer to hold — one would only be a way to get stuck. + return ctrl.Result{}, nil + } + + owned, err := r.ownedSandboxes(ctx, &set) + if err != nil { + return ctrl.Result{}, err + } + + // Prune boxes that are dead but still counting toward the total. Without this a + // set of 3 that loses one instance sits at "3 replicas, 2 usable" forever: the + // Sandbox controller will not resurrect a terminal box (a fresh instance would be + // a different box wearing the same name), so the only way back to 3 usable boxes + // is for the SET to replace it. Deleting here makes the next branch see the + // shortfall and create a replacement on this same pass. + if pruned, err := r.pruneTerminal(ctx, owned); err != nil { + return ctrl.Result{}, err + } else if pruned > 0 { + log.Info("pruned terminal sandboxes for replacement", + "sandboxset", set.Name, "count", pruned) + if owned, err = r.ownedSandboxes(ctx, &set); err != nil { + return ctrl.Result{}, err + } + } + + switch diff := int(set.Spec.Replicas) - len(owned); { + case diff > 0: + if err := r.scaleUp(ctx, &set, diff); err != nil { + return ctrl.Result{}, err + } + log.Info("created sandboxes", "sandboxset", set.Name, "count", diff) + case diff < 0: + victims := selectForRemoval(owned, -diff) + if err := r.scaleDown(ctx, victims); err != nil { + return ctrl.Result{}, err + } + log.Info("deleted sandboxes", "sandboxset", set.Name, "count", len(victims)) + } + + // Re-list after mutating so status reflects what actually exists rather than what + // we intended. A create that was rejected (quota, admission) must not be reported + // as a replica. + if owned, err = r.ownedSandboxes(ctx, &set); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.setStatus(ctx, &set, owned) +} + +// ownedSandboxes lists the Sandboxes this set owns, sorted by name so both the +// removal choice and the reported list are deterministic. +// +// It filters by ownerReference UID rather than trusting the label selector alone: +// the label is user-visible and could be applied to a foreign Sandbox, and acting +// on that would let anyone get a box they do not own deleted by our scale-in. +func (r *SandboxSetReconciler) ownedSandboxes(ctx context.Context, set *nebulav1alpha1.SandboxSet) ([]nebulav1alpha1.Sandbox, error) { + var list nebulav1alpha1.SandboxList + if err := r.List(ctx, &list, + client.InNamespace(set.Namespace), + client.MatchingLabels{nebulav1alpha1.SandboxSetLabel: set.Name}, + ); err != nil { + return nil, err + } + + owned := make([]nebulav1alpha1.Sandbox, 0, len(list.Items)) + for i := range list.Items { + sbx := list.Items[i] + if ref := metav1.GetControllerOf(&sbx); ref == nil || ref.UID != set.UID { + continue + } + if !sbx.DeletionTimestamp.IsZero() { + continue // already going away; do not count it toward the desired total + } + owned = append(owned, sbx) + } + sort.Slice(owned, func(i, j int) bool { return owned[i].Name < owned[j].Name }) + return owned, nil +} + +// scaleUp creates n new Sandboxes from the set's template. +func (r *SandboxSetReconciler) scaleUp(ctx context.Context, set *nebulav1alpha1.SandboxSet, n int) error { + for range n { + sbx := r.buildSandbox(set) + if err := controllerutil.SetControllerReference(set, sbx, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, sbx); err != nil { + // Surface the first failure rather than pressing on: if creates are being + // rejected (quota, admission), the next one fails the same way, and a partial + // scale-up plus a real error is more useful than n identical errors. + return err + } + } + return nil +} + +// buildSandbox stamps one Sandbox out of the template. The name is GENERATED +// (metadata.generateName) rather than an ordinal: an ordinal implies a slot that +// gets refilled, so a replacement box would wear a dead box's name — same address, +// different filesystem. A generated name makes a replacement visibly a new box. +func (r *SandboxSetReconciler) buildSandbox(set *nebulav1alpha1.SandboxSet) *nebulav1alpha1.Sandbox { + labelSet := map[string]string{} + for k, v := range set.Spec.Template.Metadata.Labels { + labelSet[k] = v + } + // Applied last so a template cannot overwrite the ownership label the set + // selects on — doing so would orphan the box from its own set. + labelSet[nebulav1alpha1.SandboxSetLabel] = set.Name + labelSet[nebulav1alpha1.ManagedByLabel] = nebulav1alpha1.ManagedByValue + + var annotations map[string]string + if len(set.Spec.Template.Metadata.Annotations) > 0 { + annotations = make(map[string]string, len(set.Spec.Template.Metadata.Annotations)) + for k, v := range set.Spec.Template.Metadata.Annotations { + annotations[k] = v + } + } + + return &nebulav1alpha1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: set.Name + "-", + Namespace: set.Namespace, + Labels: labelSet, + Annotations: annotations, + }, + Spec: *set.Spec.Template.Spec.DeepCopy(), + } +} + +// pruneTerminal deletes owned boxes that have reached a terminal phase, returning +// how many it removed. A terminal box holds no instance and will never come back — +// the Sandbox controller refuses to recreate one, deliberately, so that a user is +// never silently handed an empty box under the name of the one they were working in. +// Replacement is therefore the SET's job, and it starts with removing the corpse. +// +// This is also what makes TTL a recycle interval for a set and a hard deadline for a +// standalone Sandbox: same expiry, but here the set notices the shortfall and creates +// a fresh box. +func (r *SandboxSetReconciler) pruneTerminal(ctx context.Context, owned []nebulav1alpha1.Sandbox) (int, error) { + var pruned int + for i := range owned { + sbx := &owned[i] + if !isTerminalSandboxPhase(sbx.Status.Phase) { + continue + } + preconditions := metav1.Preconditions{UID: &sbx.UID} + if err := r.Delete(ctx, sbx, &client.DeleteOptions{Preconditions: &preconditions}); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + return pruned, err + } + } + pruned++ + } + return pruned, nil +} + +// scaleDown deletes the chosen boxes. +func (r *SandboxSetReconciler) scaleDown(ctx context.Context, victims []nebulav1alpha1.Sandbox) error { + for i := range victims { + v := &victims[i] + // UID-pinned so a box already replaced by a same-named recreate is never + // clobbered; an already-gone box is success. + preconditions := metav1.Preconditions{UID: &v.UID} + if err := r.Delete(ctx, v, &client.DeleteOptions{Preconditions: &preconditions}); err != nil { + // An already-gone box is success, and must NOT end the loop: returning here + // would abandon the remaining victims while reporting the scale-in as done, + // leaving paid instances running. + if err = client.IgnoreNotFound(err); err != nil { + return err + } + } + } + return nil +} + +// selectForRemoval picks which n boxes to delete on scale-in, cheapest-to-lose +// first. Scale-in has to name a victim, and the boxes are NOT interchangeable once +// someone is working in one, so the order is chosen to minimise destroyed work: +// +// 1. Terminal boxes — already worthless. pruneTerminal normally removes these +// before we get here, so this rank is a safety net for a box that turned +// terminal between the prune and this call. +// 2. Not-yet-Ready boxes — nobody can have been using a box that was never +// reachable. Youngest first, so the box closest to becoming useful survives. +// 3. Ready boxes — youngest first, on the reasoning that the most recently created +// box is the least likely to have been claimed and worked in. +// +// This is deliberately NOT StatefulSet's highest-ordinal rule, which here would +// mean "kill whichever box happens to sort last" — including one in active use +// while a failed box sits beside it. +// +// The rule we actually want for step 3 is least-recently-USED, so an idle box goes +// before one holding a live session. That needs per-box activity data, which only +// SandD can report and does not yet; "youngest Ready" is the best available proxy +// until it does. +func selectForRemoval(owned []nebulav1alpha1.Sandbox, n int) []nebulav1alpha1.Sandbox { + if n >= len(owned) { + return owned + } + + candidates := make([]nebulav1alpha1.Sandbox, len(owned)) + copy(candidates, owned) + sort.SliceStable(candidates, func(i, j int) bool { + ri, rj := removalRank(&candidates[i]), removalRank(&candidates[j]) + if ri != rj { + return ri < rj + } + // Within a rank, youngest first. + return candidates[i].CreationTimestamp.After(candidates[j].CreationTimestamp.Time) + }) + return candidates[:n] +} + +// removalRank orders boxes by how little it costs to lose them: lower goes first. +func removalRank(sbx *nebulav1alpha1.Sandbox) int { + switch sbx.Status.Phase { + case nebulav1alpha1.SandboxFailed, nebulav1alpha1.SandboxExpired: + return 0 // dead already + case nebulav1alpha1.SandboxReady: + return 2 // possibly in use — last resort + default: + return 1 // still coming up: nobody has used it yet + } +} + +// setStatus rolls the owned boxes up into the set's status, including the selector +// the /scale subresource needs. It skips the write when nothing changed, so a +// steady-state set does not generate an update per resync. +func (r *SandboxSetReconciler) setStatus(ctx context.Context, set *nebulav1alpha1.SandboxSet, owned []nebulav1alpha1.Sandbox) error { + before := set.Status.DeepCopy() + + var ready int32 + names := make([]string, 0, len(owned)) + for i := range owned { + names = append(names, owned[i].Name) + if owned[i].Status.Phase == nebulav1alpha1.SandboxReady { + ready++ + } + } + + set.Status.Replicas = int32(len(owned)) + set.Status.ReadyReplicas = ready + set.Status.Sandboxes = names + // The /scale subresource requires the selector as a serialized string; HPA reads + // it from here to find the set's members, so autoscaling silently does nothing + // without it. + set.Status.Selector = labels.SelectorFromSet(labels.Set{ + nebulav1alpha1.SandboxSetLabel: set.Name, + }).String() + + reason, msg, condStatus := nebulav1alpha1.ReasonSandboxSetProgressing, + "waiting for sandboxes to become ready", metav1.ConditionFalse + switch { + case set.Spec.Replicas == 0: + reason, msg = nebulav1alpha1.ReasonSandboxSetScaledToZero, "scaled to zero" + case ready == set.Spec.Replicas: + reason, msg, condStatus = nebulav1alpha1.ReasonSandboxSetReady, + "all sandboxes are ready", metav1.ConditionTrue + } + apimeta.SetStatusCondition(&set.Status.Conditions, metav1.Condition{ + Type: nebulav1alpha1.SandboxSetConditionReady, + Status: condStatus, + Reason: reason, + Message: msg, + ObservedGeneration: set.Generation, + }) + + if apiequality.Semantic.DeepEqual(before, &set.Status) { + return nil + } + return r.Status().Update(ctx, set) +} + +// SetupWithManager wires the controller. It owns Sandboxes, so a box becoming +// ready or failing re-reconciles the set immediately — which is what makes +// self-healing prompt: a Failed box is removed and replaced on that same event +// rather than at the next resync. +func (r *SandboxSetReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nebulav1alpha1.SandboxSet{}). + Owns(&nebulav1alpha1.Sandbox{}). + Named("sandboxset"). + Complete(r) +} diff --git a/internal/controller/sandboxset_controller_test.go b/internal/controller/sandboxset_controller_test.go new file mode 100644 index 0000000..690785d --- /dev/null +++ b/internal/controller/sandboxset_controller_test.go @@ -0,0 +1,381 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +const testSetName = "workers" + +func newSetReconciler(objs ...client.Object) (*SandboxSetReconciler, client.Client) { + s := runtime.NewScheme() + _ = clientgoscheme.AddToScheme(s) + _ = nebulav1alpha1.AddToScheme(s) + c := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&nebulav1alpha1.SandboxSet{}, &nebulav1alpha1.Sandbox{}). + Build() + return &SandboxSetReconciler{Client: c, Scheme: s}, c +} + +func newSandboxSet(replicas int32) *nebulav1alpha1.SandboxSet { + return &nebulav1alpha1.SandboxSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSetName, + Namespace: testNS, + UID: types.UID("set-uid-1"), + }, + Spec: nebulav1alpha1.SandboxSetSpec{ + Replicas: replicas, + Template: nebulav1alpha1.SandboxTemplateSpec{ + Metadata: nebulav1alpha1.SandboxTemplateMetadata{ + Labels: map[string]string{"tier": "agent"}, + }, + Spec: nebulav1alpha1.SandboxSpec{ + NodePoolRef: "gpu", + Image: "ubuntu:24.04", + }, + }, + }, + } +} + +func reconcileSet(t *testing.T, r *SandboxSetReconciler) { + t.Helper() + if _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: testNS, Name: testSetName}, + }); err != nil { + t.Fatalf("Reconcile: %v", err) + } +} + +func listSandboxes(t *testing.T, c client.Client) []nebulav1alpha1.Sandbox { + t.Helper() + var list nebulav1alpha1.SandboxList + if err := c.List(context.Background(), &list, client.InNamespace(testNS)); err != nil { + t.Fatalf("list sandboxes: %v", err) + } + return list.Items +} + +func getSet(t *testing.T, c client.Client) *nebulav1alpha1.SandboxSet { + t.Helper() + var set nebulav1alpha1.SandboxSet + key := client.ObjectKey{Namespace: testNS, Name: testSetName} + if err := c.Get(context.Background(), key, &set); err != nil { + t.Fatalf("get sandboxset: %v", err) + } + return &set +} + +// ownedSandbox is a member box as the set would have created it, with the phase +// and age a test needs. The UID is explicit because ownership is matched on it. +func ownedSandbox(name string, phase nebulav1alpha1.SandboxPhase, ageMinutes int) *nebulav1alpha1.Sandbox { + yes := true + return &nebulav1alpha1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNS, + UID: types.UID("uid-" + name), + CreationTimestamp: metav1.NewTime( + time.Now().Add(-time.Duration(ageMinutes) * time.Minute)), + Labels: map[string]string{nebulav1alpha1.SandboxSetLabel: testSetName}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nebulav1alpha1.GroupVersion.String(), + Kind: "SandboxSet", + Name: testSetName, + UID: types.UID("set-uid-1"), + Controller: &yes, + }}, + }, + Spec: nebulav1alpha1.SandboxSpec{NodePoolRef: "gpu", Image: "ubuntu:24.04"}, + Status: nebulav1alpha1.SandboxStatus{Phase: phase}, + } +} + +// TestSandboxSetScalesUpFromTemplate: the set must create exactly Replicas boxes, +// each stamped from the template and labelled so the set can find it again. +func TestSandboxSetScalesUpFromTemplate(t *testing.T) { + r, c := newSetReconciler(newSandboxSet(3)) + reconcileSet(t, r) + + boxes := listSandboxes(t, c) + if len(boxes) != 3 { + t.Fatalf("sandboxes = %d, want 3", len(boxes)) + } + for i := range boxes { + sbx := &boxes[i] + if sbx.Spec.Image != "ubuntu:24.04" || sbx.Spec.NodePoolRef != "gpu" { + t.Errorf("%s: spec not stamped from the template: %+v", sbx.Name, sbx.Spec) + } + if got := sbx.Labels[nebulav1alpha1.SandboxSetLabel]; got != testSetName { + t.Errorf("%s: ownership label = %q, want %q", sbx.Name, got, testSetName) + } + if got := sbx.Labels["tier"]; got != "agent" { + t.Errorf("%s: template label lost: tier = %q", sbx.Name, got) + } + if ref := metav1.GetControllerOf(sbx); ref == nil || ref.Kind != "SandboxSet" { + t.Errorf("%s: controller ref = %+v, want the SandboxSet", sbx.Name, ref) + } + } +} + +// TestSandboxSetTemplateCannotOrphanBox: the ownership label is applied after the +// template's, so a template that sets it cannot detach the box from its own set — +// which would leave an unowned box billing forever, invisible to the set. +func TestSandboxSetTemplateCannotOrphanBox(t *testing.T) { + set := newSandboxSet(1) + set.Spec.Template.Metadata.Labels[nebulav1alpha1.SandboxSetLabel] = "somewhere-else" + r, c := newSetReconciler(set) + reconcileSet(t, r) + + boxes := listSandboxes(t, c) + if len(boxes) != 1 { + t.Fatalf("sandboxes = %d, want 1", len(boxes)) + } + if got := boxes[0].Labels[nebulav1alpha1.SandboxSetLabel]; got != testSetName { + t.Errorf("ownership label = %q, want %q: the template must not override it", got, testSetName) + } +} + +// TestSandboxSetIsIdempotent: re-reconciling a satisfied set must not create more +// boxes. Each box is a paid instance, so a leak here is a bill, not just a bug. +func TestSandboxSetIsIdempotent(t *testing.T) { + r, c := newSetReconciler(newSandboxSet(2)) + reconcileSet(t, r) + reconcileSet(t, r) + reconcileSet(t, r) + + if n := len(listSandboxes(t, c)); n != 2 { + t.Errorf("sandboxes = %d, want 2: reconcile must be idempotent", n) + } +} + +// TestSandboxSetIgnoresForeignSandbox: a box carrying the set's label but owned by +// someone else must be neither counted nor deleted. Counting it would starve the +// set; deleting it would let anyone destroy a box they do not own by labelling it. +func TestSandboxSetIgnoresForeignSandbox(t *testing.T) { + foreign := ownedSandbox("imposter", nebulav1alpha1.SandboxReady, 5) + foreign.OwnerReferences = nil + + r, c := newSetReconciler(newSandboxSet(1), foreign) + reconcileSet(t, r) + + boxes := listSandboxes(t, c) + if len(boxes) != 2 { + t.Fatalf("sandboxes = %d, want 2 (the imposter plus one real box)", len(boxes)) + } + var found bool + for i := range boxes { + if boxes[i].Name == "imposter" { + found = true + } + } + if !found { + t.Error("the foreign Sandbox was deleted; a label alone must not grant the set authority over it") + } + if got := getSet(t, c).Status.Replicas; got != 1 { + t.Errorf("status.replicas = %d, want 1: the foreign box must not be counted", got) + } +} + +// TestSandboxSetReplacesTerminalBox is the self-healing path: a terminal box is +// deleted AND replaced on the same pass. Without the prune the set would sit at +// "3 replicas, 2 usable" forever, because the Sandbox controller deliberately never +// resurrects a dead box. +func TestSandboxSetReplacesTerminalBox(t *testing.T) { + for _, phase := range []nebulav1alpha1.SandboxPhase{ + nebulav1alpha1.SandboxFailed, + nebulav1alpha1.SandboxExpired, + } { + t.Run(string(phase), func(t *testing.T) { + dead := ownedSandbox("workers-dead", phase, 30) + alive := ownedSandbox("workers-alive", nebulav1alpha1.SandboxReady, 20) + + r, c := newSetReconciler(newSandboxSet(2), dead, alive) + reconcileSet(t, r) + + boxes := listSandboxes(t, c) + if len(boxes) != 2 { + t.Fatalf("sandboxes = %d, want 2", len(boxes)) + } + for i := range boxes { + if boxes[i].Name == "workers-dead" { + t.Error("the terminal box was not pruned; the set can never return to 2 usable boxes") + } + } + if got := getSet(t, c).Status.Replicas; got != 2 { + t.Errorf("status.replicas = %d, want 2", got) + } + }) + } +} + +// TestSandboxSetScalesDown checks scale-in removes exactly the excess. +func TestSandboxSetScalesDown(t *testing.T) { + objs := []client.Object{newSandboxSet(1)} + for i := range 3 { + objs = append(objs, ownedSandbox(fmt.Sprintf("workers-%d", i), + nebulav1alpha1.SandboxReady, 10+i)) + } + r, c := newSetReconciler(objs...) + reconcileSet(t, r) + + boxes := listSandboxes(t, c) + if len(boxes) != 1 { + t.Fatalf("sandboxes = %d, want 1", len(boxes)) + } + // Youngest-first within the Ready rank: workers-0 is the youngest (10m) and + // workers-2 the oldest (12m), so the OLDEST box is the survivor. + if boxes[0].Name != "workers-2" { + t.Errorf("survivor = %q, want workers-2 (the oldest Ready box)", boxes[0].Name) + } +} + +// TestSandboxSetScaleDownRemovesAllExcess: scale-in from 3 to 0 must delete every +// box in ONE pass. Bailing out early would leave paid instances running while the +// set reported the scale-in as done. +func TestSandboxSetScaleDownRemovesAllExcess(t *testing.T) { + objs := []client.Object{newSandboxSet(0)} + for i := range 3 { + objs = append(objs, ownedSandbox(fmt.Sprintf("workers-%d", i), + nebulav1alpha1.SandboxReady, 10+i)) + } + r, c := newSetReconciler(objs...) + reconcileSet(t, r) + + if n := len(listSandboxes(t, c)); n != 0 { + t.Errorf("sandboxes = %d, want 0: every excess box must be deleted in one pass", n) + } + set := getSet(t, c) + if set.Status.Replicas != 0 { + t.Errorf("status.replicas = %d, want 0", set.Status.Replicas) + } + if reason := readyReason(set.Status.Conditions); reason != nebulav1alpha1.ReasonSandboxSetScaledToZero { + t.Errorf("condition reason = %q, want %q", reason, nebulav1alpha1.ReasonSandboxSetScaledToZero) + } +} + +// TestSelectForRemovalOrder pins the victim order directly, since it decides whose +// work gets destroyed: dead boxes first, then boxes nobody could have used, and a +// possibly-in-use Ready box only as a last resort. +func TestSelectForRemovalOrder(t *testing.T) { + ready := *ownedSandbox("ready", nebulav1alpha1.SandboxReady, 30) + provisioning := *ownedSandbox("provisioning", nebulav1alpha1.SandboxProvisioning, 20) + failed := *ownedSandbox("failed", nebulav1alpha1.SandboxFailed, 10) + owned := []nebulav1alpha1.Sandbox{ready, provisioning, failed} + + got := selectForRemoval(owned, 2) + if len(got) != 2 { + t.Fatalf("victims = %d, want 2", len(got)) + } + if got[0].Name != "failed" { + t.Errorf("first victim = %q, want failed (a dead box costs nothing to lose)", got[0].Name) + } + if got[1].Name != "provisioning" { + t.Errorf("second victim = %q, want provisioning (nobody can have used it)", got[1].Name) + } + + // Asking for more than exists must return everything, not panic on a slice bound. + if n := len(selectForRemoval(owned, 5)); n != 3 { + t.Errorf("victims = %d, want 3 when n exceeds the population", n) + } +} + +// TestSandboxSetStatusSelector: /scale requires the selector as a serialized +// string, and HPA reads it from status to find the set's members — autoscaling +// silently does nothing if it is wrong. +func TestSandboxSetStatusSelector(t *testing.T) { + r, c := newSetReconciler(newSandboxSet(1)) + reconcileSet(t, r) + + want := nebulav1alpha1.SandboxSetLabel + "=" + testSetName + if got := getSet(t, c).Status.Selector; got != want { + t.Errorf("status.selector = %q, want %q", got, want) + } +} + +// TestSandboxSetReadyRollup: the set is Ready only when every box is, and the +// reported names must be the boxes that actually exist. +func TestSandboxSetReadyRollup(t *testing.T) { + a := ownedSandbox("workers-a", nebulav1alpha1.SandboxReady, 10) + b := ownedSandbox("workers-b", nebulav1alpha1.SandboxProvisioning, 5) + r, c := newSetReconciler(newSandboxSet(2), a, b) + reconcileSet(t, r) + + set := getSet(t, c) + if set.Status.Replicas != 2 || set.Status.ReadyReplicas != 1 { + t.Errorf("replicas/ready = %d/%d, want 2/1", set.Status.Replicas, set.Status.ReadyReplicas) + } + if readyCondStatus(set.Status.Conditions) != metav1.ConditionFalse { + t.Error("Ready must be False while one box is still coming up") + } + if len(set.Status.Sandboxes) != 2 { + t.Errorf("status.sandboxes = %v, want both box names", set.Status.Sandboxes) + } + + // Bring the laggard up: the set must flip to Ready. + b.Status.Phase = nebulav1alpha1.SandboxReady + if err := c.Status().Update(context.Background(), b); err != nil { + t.Fatalf("update sandbox status: %v", err) + } + reconcileSet(t, r) + + set = getSet(t, c) + if set.Status.ReadyReplicas != 2 { + t.Errorf("readyReplicas = %d, want 2", set.Status.ReadyReplicas) + } + if readyCondStatus(set.Status.Conditions) != metav1.ConditionTrue { + t.Error("Ready must be True once every box is ready") + } + if reason := readyReason(set.Status.Conditions); reason != nebulav1alpha1.ReasonSandboxSetReady { + t.Errorf("condition reason = %q, want %q", reason, nebulav1alpha1.ReasonSandboxSetReady) + } +} + +func readyCondStatus(conds []metav1.Condition) metav1.ConditionStatus { + for _, c := range conds { + if c.Type == nebulav1alpha1.SandboxSetConditionReady { + return c.Status + } + } + return "" +} + +func readyReason(conds []metav1.Condition) string { + for _, c := range conds { + if c.Type == nebulav1alpha1.SandboxSetConditionReady { + return c.Reason + } + } + return "" +} diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go index 74a75dc..3c43a80 100644 --- a/internal/webhook/v1/pod_webhook.go +++ b/internal/webhook/v1/pod_webhook.go @@ -74,7 +74,7 @@ func (d *PodCustomDefaulter) Default(_ context.Context, obj runtime.Object) erro return fmt.Errorf("expected a Pod object but got %T", obj) } - if pod.Labels[nebulav1alpha1.EnabledLabel] != "true" { + if pod.Labels[nebulav1alpha1.EnabledLabel] != nebulav1alpha1.EnabledValue { return nil // not opted in; leave the Pod untouched } if pod.Spec.NodeName != "" { diff --git a/pkg/cert/cert.go b/pkg/cert/cert.go new file mode 100644 index 0000000..eeaa4fb --- /dev/null +++ b/pkg/cert/cert.go @@ -0,0 +1,140 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cert provisions the webhook serving certificate in-process, so Nebula +// has no cert-manager dependency and no out-of-band setup step. +// +// The webhook needs two things that must agree exactly: a TLS keypair the manager +// serves from, and that cert's CA in the MutatingWebhookConfiguration's caBundle +// so the API server trusts the webhook when it calls it. There were two prior ways +// to get them, and both are worse: +// +// - cert-manager: rotates correctly, but is a whole second operator the user must +// install before Nebula works at all. It also made the e2e suite install +// cert-manager in BeforeSuite, which is why that suite could not run without +// network access to fetch its manifests. +// - hack/gen-webhook-cert.sh: no dependency, but the cert is minted by a shell +// script at deploy time and NEVER rotates. It is valid for CERT_DAYS (default +// 3650) and renewing means re-running the script with FORCE_REGEN — i.e. the +// expiry is a silent time bomb that fires years later, when nobody remembers +// the script exists. +// +// The rotator does both jobs in-process: it mints the keypair into a Secret, patches +// the caBundle, and then keeps renewing before expiry. Because the CA is patched from +// the same cert that was just written, the served cert and the trusted CA cannot drift. +// +// The Secret is the ONLY thing it writes. It never touches the filesystem — the +// keypair reaches the webhook server's certDir because the manager projects that +// Secret there as a volume, and the rotator merely polls the path to know when the +// kubelet has done so. See certDir below; getting this backwards (an emptyDir at +// certDir) silently prevents the whole manager from starting. +package cert + +import ( + "fmt" + + rotator "github.com/open-policy-agent/cert-controller/pkg/rotator" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" +) + +const ( + // serviceName is the webhook Service the cert is issued for. The rotator needs + // it to build the DNS name the API server dials, so it must match + // config/webhook/service.yaml AFTER kustomize applies the nebula- namePrefix. + serviceName = "nebula-webhook-service" + + // secretName is the Secret the keypair is stored in. Like the two names above it + // carries the nebula- prefix, because that is what config/default's namePrefix + // actually renders ("webhook-server-cert" here would look for a Secret no overlay + // creates). + // + // The Secret MUST already exist when the manager starts, which is why + // config/webhook ships it empty: the rotator Gets this Secret and then Updates it, + // but never Creates it, so an absent Secret is a fatal startup error that + // crash-loops the manager rather than something it recovers from. Empty is all it + // needs — nil Data is exactly the condition that triggers minting. + secretName = "nebula-webhook-server-cert" + + // certDir is where the keypair lands on disk and where controller-runtime's + // webhook server reads it from. It is the controller-runtime default, and the path + // the manager projects the Secret above at. + // + // The rotator does NOT write here, despite the name: cert-controller performs no + // filesystem writes at all, and CertDir is a path it only os.Stat()s to decide + // readiness (ensureCertsMounted). The KUBELET puts the files here by projecting the + // Secret — so the volume must be that Secret, not an emptyDir. With an emptyDir the + // files never appear, IsReady never closes, and since controller and webhook + // registration waits on it (see CertsManager), nothing ever starts while the manager + // still reports Running. + certDir = "/tmp/k8s-webhook-server/serving-certs" + + // mutatingWebhookConfName is the MutatingWebhookConfiguration whose caBundle + // gets patched — the nebula- prefixed name from config/webhook/manifests.yaml. + // + // There is deliberately no ValidatingWebhookConfiguration here: Nebula has only + // the Pod defaulter (see internal/webhook/v1), and Sandbox validation is done + // with CEL in the CRD rather than a webhook. Naming a config that does not exist + // would make the rotator fail to patch it on every reconcile. + mutatingWebhookConfName = "nebula-mutating-webhook-configuration" + + caName = "nebula-ca" + caOrg = "nebula" +) + +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update + +// CertsManager registers the cert rotator with the manager. It closes setupFinish +// once the cert is on disk and the caBundle is patched. +// +// Nothing that depends on the webhook may start before that channel closes: the +// webhook server would otherwise serve on a missing keypair and the API server +// would reject the call, which — with failurePolicy=Fail — means every Pod CREATE +// in the cluster fails admission. The caller runs controller setup in a goroutine +// blocked on this channel (see cmd/main.go). +// +// namespace must be the namespace the manager actually runs in, since it scopes +// both the Secret and the cert's DNS name; it is read from POD_NAMESPACE rather +// than hardcoded so a non-default install namespace still gets a valid cert. +func CertsManager(mgr ctrl.Manager, namespace string, setupFinish chan struct{}) error { + // The DNS name the API server dials, and therefore the name the cert must be + // valid for: ..svc. + dnsName := fmt.Sprintf("%s.%s.svc", serviceName, namespace) + + return rotator.AddRotator(mgr, &rotator.CertRotator{ + SecretKey: types.NamespacedName{ + Namespace: namespace, + Name: secretName, + }, + CertDir: certDir, + CAName: caName, + CAOrganization: caOrg, + DNSName: dnsName, + IsReady: setupFinish, + Webhooks: []rotator.WebhookInfo{{ + Type: rotator.Mutating, + Name: mutatingWebhookConfName, + }}, + // RequireLeaderElection is deliberately left false. The rotator must run in + // EVERY replica, not just the leader: CertDir is each pod's own local disk, and a + // replica that never wrote the keypair there cannot serve the webhook — and + // webhook serving is not leader-elected, so the API server will call a + // non-leader. The Secret is the shared source of truth, so replicas after the + // first find a valid cert there and simply write it to their own disk rather than + // minting a competing one. + }) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 5ef295b..b8a0ed7 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -18,7 +18,6 @@ package e2e import ( "fmt" - "os" "os/exec" "testing" @@ -28,24 +27,18 @@ import ( "github.com/InftyAI/Nebula/test/utils" ) -var ( - // Optional Environment Variables: - // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. - // These variables are useful if CertManager is already installed, avoiding - // re-installation and conflicts. - skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" - // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster - isCertManagerAlreadyInstalled = false +// projectImage is the name of the image which will be build and loaded +// with the code source changes to be tested. +const projectImage = "example.com/nebula:v0.0.1" - // projectImage is the name of the image which will be build and loaded - // with the code source changes to be tested. - projectImage = "example.com/nebula:v0.0.1" -) +// NOTE: there is deliberately no cert-manager setup here. The manager provisions its +// own webhook serving cert in-process (pkg/cert), so this suite needs neither a +// cert-manager install nor the network access to fetch its manifests — which is what +// used to make BeforeSuite fail on a machine without them. // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, // temporary environment to validate project changes with the purpose of being used in CI jobs. -// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs -// CertManager. +// The default setup requires Kind and builds/loads the Manager Docker image locally. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) _, _ = fmt.Fprintf(GinkgoWriter, "Starting nebula integration test suite\n") @@ -63,27 +56,4 @@ var _ = BeforeSuite(func() { By("loading the manager(Operator) image on Kind") err = utils.LoadImageToKindClusterWithName(projectImage) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") - - // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. - // To prevent errors when tests run in environments with CertManager already installed, - // we check for its presence before execution. - // Setup CertManager before the suite if not skipped and if not already installed - if !skipCertManagerInstall { - By("checking if cert manager is installed already") - isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() - if !isCertManagerAlreadyInstalled { - _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") - Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") - } else { - _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") - } - } -}) - -var _ = AfterSuite(func() { - // Teardown CertManager after the suite if not skipped and if it was not already installed - if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { - _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") - utils.UninstallCertManager() - } }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index f5b2d29..e26b402 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -81,15 +81,10 @@ var _ = Describe("Manager", Ordered, func() { _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - // The manager mounts webhook-server-cert as a REQUIRED volume, so the - // self-signed cert Secret must exist before the pod starts, or it wedges - // in ContainerCreating (Pending) forever. Nebula does not use cert-manager - // (see config/default/kustomization.yaml); hack/gen-webhook-cert.sh is the - // source of truth, matching the ordering hack/deploy.sh uses in prod. - By("provisioning the webhook serving certificate Secret") - cmd = exec.Command("hack/gen-webhook-cert.sh", "secret") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to provision the webhook serving cert Secret") + // No cert step here: the manager mints its own webhook serving cert at + // startup (pkg/cert) into an emptyDir, so there is nothing to pre-create — + // the same ordering hack/deploy.sh uses in prod. The cert only exists once + // the manager is RUNNING, so the assertions below must be Eventually. // Deploy via the e2e overlay, which bakes NEBULA_ENABLE_FAKE_PROVIDER=true // into the manager env so the in-memory fake provider registers at first @@ -104,13 +99,10 @@ var _ = Describe("Manager", Ordered, func() { _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") - // caBundle injection is server-side (only the API server reads it) and - // requires the MutatingWebhookConfiguration created by the deploy, so it - // runs after the deploy. The manager pod is untouched — no restart needed. - By("injecting the webhook CA bundle") - cmd = exec.Command("hack/gen-webhook-cert.sh", "cabundle") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to inject the webhook CA bundle") + // No caBundle injection step: the manager patches the + // MutatingWebhookConfiguration itself once its cert rotator runs (pkg/cert), + // from the same cert it just wrote. The "CA injection" spec below asserts that + // happened, so this is covered by an Eventually rather than a deploy step. }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, @@ -306,11 +298,11 @@ var _ = Describe("Manager", Ordered, func() { }) It("should have the self-signed webhook serving cert Secret", func() { - // Nebula does not use cert-manager; hack/gen-webhook-cert.sh creates this - // self-signed Secret in the e2e BeforeAll (and hack/deploy.sh in prod). + // The manager's own cert rotator (pkg/cert) creates this Secret after it + // starts — nothing pre-creates it, so this asserts the rotator ran. By("validating that the webhook serving cert Secret exists") verifyCertSecret := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + cmd := exec.Command("kubectl", "get", "secrets", "nebula-webhook-server-cert", "-n", namespace) _, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) } diff --git a/test/utils/utils.go b/test/utils/utils.go index 8ae13e7..0d33551 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -31,9 +31,6 @@ const ( prometheusOperatorVersion = "v0.77.1" prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + "releases/download/%s/bundle.yaml" - - certmanagerVersion = "v1.16.3" - certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" ) func warnError(err error) { @@ -106,66 +103,9 @@ func IsPrometheusCRDsInstalled() bool { return false } -// UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { - return err - } - // Wait for cert-manager-webhook to be ready, which can take time if cert-manager - // was re-installed after uninstalling on a cluster. - cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", - "--for", "condition=Available", - "--namespace", "cert-manager", - "--timeout", "5m", - ) - - _, err := Run(cmd) - return err -} - -// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed -// by verifying the existence of key CRDs related to Cert Manager. -func IsCertManagerCRDsInstalled() bool { - // List of common Cert Manager CRDs - certManagerCRDs := []string{ - "certificates.cert-manager.io", - "issuers.cert-manager.io", - "clusterissuers.cert-manager.io", - "certificaterequests.cert-manager.io", - "orders.acme.cert-manager.io", - "challenges.acme.cert-manager.io", - } - - // Execute the kubectl command to get all CRDs - cmd := exec.Command("kubectl", "get", "crds") - output, err := Run(cmd) - if err != nil { - return false - } - - // Check if any of the Cert Manager CRDs are present - crdList := GetNonEmptyLines(output) - for _, crd := range certManagerCRDs { - for _, line := range crdList { - if strings.Contains(line, crd) { - return true - } - } - } - - return false -} +// No cert-manager helpers here, deliberately: the manager provisions its own +// webhook serving cert in-process (pkg/cert), so the e2e suite has nothing to +// install or wait for before deploying. // LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error {