diff --git a/api/core/v1beta1/conditions.go b/api/core/v1beta1/conditions.go index 8cf172687..6f8a3b257 100644 --- a/api/core/v1beta1/conditions.go +++ b/api/core/v1beta1/conditions.go @@ -605,4 +605,12 @@ const ( // OpenStackVersionMinorUpdateAvailableMessage OpenStackVersionMinorUpdateAvailableMessage = "update available" + + // OpenStackVersionMinorUpdateReadyGatedMessage - format string; arg is the target stage name + OpenStackVersionMinorUpdateReadyGatedMessage = "Minor update progression stopped after stage: %s. Set annotation to any stage after %s to resume OpenStack update or remove the annotation to run to completion." ) + +// OpenStackVersionMinorUpdateGatedReason is set on the next stage condition when the +// target-stage annotation pauses minor update progression. It must not be reused for +// in-progress work (see condition.RequestedReason). +const OpenStackVersionMinorUpdateGatedReason condition.Reason = "Gated" diff --git a/api/core/v1beta1/openstackversion_types.go b/api/core/v1beta1/openstackversion_types.go index fea75a03f..21c433f83 100644 --- a/api/core/v1beta1/openstackversion_types.go +++ b/api/core/v1beta1/openstackversion_types.go @@ -34,8 +34,128 @@ const ( MinorUpdateControlPlane string = "Minor Update Controlplane In Progress" // MinorUpdateComplete - MinorUpdateComplete string = "Complete" + + // MinorUpdateTargetStageAnnotation - specifies the update stage after which the minor update + // should pause. All stages up to and including the named stage will be completed; subsequent + // stages will be blocked until the annotation is removed or updated to a later stage. + // During an update, the webhook rejects moving this annotation to an earlier stage, and + // rejects adding it behind stages already completed when it was absent at update start. + // Valid values: "ovn-controlplane", "ovn-dataplane", "rabbitmq", "mariadb", "memcached", + // "keystone", "controlplane". Remove the annotation to let the update proceed to completion. + MinorUpdateTargetStageAnnotation string = "core.openstack.org/update-target-stage" + + // MinorUpdateStageOVNControlplane - stage name for OVN controlplane update + MinorUpdateStageOVNControlplane string = "ovn-controlplane" + // MinorUpdateStageOVNDataplane - stage name for OVN dataplane update + MinorUpdateStageOVNDataplane string = "ovn-dataplane" + // MinorUpdateStageRabbitMQ - stage name for RabbitMQ update + MinorUpdateStageRabbitMQ string = "rabbitmq" + // MinorUpdateStageMariaDB - stage name for MariaDB update + MinorUpdateStageMariaDB string = "mariadb" + // MinorUpdateStageMemcached - stage name for Memcached update + MinorUpdateStageMemcached string = "memcached" + // MinorUpdateStageKeystone - stage name for Keystone update + MinorUpdateStageKeystone string = "keystone" + // MinorUpdateStageControlplane - stage name for full controlplane update + MinorUpdateStageControlplane string = "controlplane" ) +// validMinorUpdateTargetStagesOrdered is the single source of truth for allowed +// MinorUpdateTargetStageAnnotation values, listed in rollout order. +var validMinorUpdateTargetStagesOrdered = []string{ + MinorUpdateStageOVNControlplane, + MinorUpdateStageOVNDataplane, + MinorUpdateStageRabbitMQ, + MinorUpdateStageMariaDB, + MinorUpdateStageMemcached, + MinorUpdateStageKeystone, + MinorUpdateStageControlplane, +} + +// minorUpdateTargetStageConditionTypes maps each validMinorUpdateTargetStagesOrdered entry to its status condition. +var minorUpdateTargetStageConditionTypes = map[string]condition.Type{ + MinorUpdateStageOVNControlplane: OpenStackVersionMinorUpdateOVNControlplane, + MinorUpdateStageOVNDataplane: OpenStackVersionMinorUpdateOVNDataplane, + MinorUpdateStageRabbitMQ: OpenStackVersionMinorUpdateRabbitMQ, + MinorUpdateStageMariaDB: OpenStackVersionMinorUpdateMariaDB, + MinorUpdateStageMemcached: OpenStackVersionMinorUpdateMemcached, + MinorUpdateStageKeystone: OpenStackVersionMinorUpdateKeystone, + MinorUpdateStageControlplane: OpenStackVersionMinorUpdateControlplane, +} + +// validMinorUpdateTargetStages is a set derived from validMinorUpdateTargetStagesOrdered for O(1) lookup. +var validMinorUpdateTargetStages = func() map[string]struct{} { + m := make(map[string]struct{}, len(validMinorUpdateTargetStagesOrdered)) + for _, s := range validMinorUpdateTargetStagesOrdered { + m[s] = struct{}{} + } + return m +}() + +// IsValidMinorUpdateTargetStage reports whether v is a supported minor-update target stage name. +func IsValidMinorUpdateTargetStage(v string) bool { + if v == "" { + return false + } + _, ok := validMinorUpdateTargetStages[v] + return ok +} + +// ValidMinorUpdateTargetStages returns allowed annotation values in rollout order. +func ValidMinorUpdateTargetStages() []string { + return append([]string(nil), validMinorUpdateTargetStagesOrdered...) +} + +// MinorUpdateTargetStageIndex returns the rollout order index for stage. +func MinorUpdateTargetStageIndex(stage string) (int, bool) { + for i, s := range validMinorUpdateTargetStagesOrdered { + if s == stage { + return i, true + } + } + return -1, false +} + +// MinorUpdateTargetStageFromAnnotations returns the target-stage annotation value when set and valid. +func MinorUpdateTargetStageFromAnnotations(annotations map[string]string) (string, bool) { + if annotations == nil { + return "", false + } + stage, ok := annotations[MinorUpdateTargetStageAnnotation] + if !ok || !IsValidMinorUpdateTargetStage(stage) { + return "", false + } + return stage, true +} + +// MinorUpdateStageAllowedForReconcile reports whether the control plane may patch resources +// for rollout stage during a minor update. When the target-stage annotation is absent, all +// stages are allowed. When set, only stages up to and including the target may be reconciled. +func MinorUpdateStageAllowedForReconcile(annotations map[string]string, stage string) bool { + target, ok := MinorUpdateTargetStageFromAnnotations(annotations) + if !ok { + return true + } + stageIdx, okStage := MinorUpdateTargetStageIndex(stage) + targetIdx, okTarget := MinorUpdateTargetStageIndex(target) + if !okStage || !okTarget { + return true + } + return stageIdx <= targetIdx +} + +// LatestCompletedMinorUpdateTargetStageIndex returns the rollout index of the furthest +// minor-update stage marked True in status, or -1 when no annotated rollout stage has completed. +func LatestCompletedMinorUpdateTargetStageIndex(status OpenStackVersionStatus) int { + latest := -1 + for i, stage := range validMinorUpdateTargetStagesOrdered { + if status.Conditions.IsTrue(minorUpdateTargetStageConditionTypes[stage]) { + latest = i + } + } + return latest +} + // OpenStackVersionSpec - defines the desired state of OpenStackVersion type OpenStackVersionSpec struct { diff --git a/api/core/v1beta1/openstackversion_webhook.go b/api/core/v1beta1/openstackversion_webhook.go index 97de0b774..c55a58a80 100644 --- a/api/core/v1beta1/openstackversion_webhook.go +++ b/api/core/v1beta1/openstackversion_webhook.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "reflect" + "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" @@ -74,6 +75,10 @@ func (r *OpenStackVersion) ValidateCreate(ctx context.Context, c goClient.Client ) } + if err := validateMinorUpdateTargetStageAnnotation(r.Annotations, r.GetName()); err != nil { + return nil, err + } + versionList, err := GetOpenStackVersions(r.Namespace, c) if err != nil { @@ -114,6 +119,10 @@ func (r *OpenStackVersion) ValidateCreate(ctx context.Context, c goClient.Client func (r *OpenStackVersion) ValidateUpdate(ctx context.Context, old runtime.Object, c goClient.Client) (admission.Warnings, error) { openstackversionlog.Info("validate update", "name", r.Name) + if err := validateMinorUpdateTargetStageAnnotation(r.Annotations, r.GetName()); err != nil { + return nil, err + } + _, ok := r.Status.ContainerImageVersionDefaults[r.Spec.TargetVersion] if r.Spec.TargetVersion != openstackVersionDefaults.AvailableVersion && !ok { return nil, apierrors.NewForbidden( @@ -135,6 +144,11 @@ func (r *OpenStackVersion) ValidateUpdate(ctx context.Context, old runtime.Objec return nil, apierrors.NewInternalError(fmt.Errorf("failed to convert old object to OpenStackVersion")) } + // Validate that the target stage annotation is not from earlier stage while a minor update is in progress + if err := validateMinorUpdateTargetStageAnnotationProgress(oldVersion, r); err != nil { + return nil, err + } + // Check if targetVersion is changing and this is a minor update if oldVersion.Spec.TargetVersion != r.Spec.TargetVersion && oldVersion.Status.DeployedVersion != nil { // Check if the skip annotation is present @@ -174,6 +188,113 @@ func (r *OpenStackVersion) ValidateUpdate(ctx context.Context, old runtime.Objec return nil, nil } +func validateMinorUpdateTargetStageAnnotation(annotations map[string]string, resourceName string) error { + if annotations == nil { + return nil + } + stage, ok := annotations[MinorUpdateTargetStageAnnotation] + if !ok { + return nil + } + annotationField := "metadata.annotations[" + MinorUpdateTargetStageAnnotation + "]" + if stage == "" { + return apierrors.NewForbidden( + schema.GroupResource{ + Group: GroupVersion.WithKind("OpenStackVersion").Group, + Resource: GroupVersion.WithKind("OpenStackVersion").Kind, + }, resourceName, &field.Error{ + Type: field.ErrorTypeForbidden, + Field: annotationField, + BadValue: stage, + Detail: "Annotation value must not be empty. Remove the annotation or set a valid stage name", + }, + ) + } + if !IsValidMinorUpdateTargetStage(stage) { + return apierrors.NewForbidden( + schema.GroupResource{ + Group: GroupVersion.WithKind("OpenStackVersion").Group, + Resource: GroupVersion.WithKind("OpenStackVersion").Kind, + }, resourceName, &field.Error{ + Type: field.ErrorTypeForbidden, + Field: annotationField, + BadValue: stage, + Detail: fmt.Sprintf( + "Invalid target stage %q. Must be one of: %s", + stage, + strings.Join(ValidMinorUpdateTargetStages(), ", "), + ), + }, + ) + } + return nil +} + +func minorUpdateInProgress(v *OpenStackVersion) bool { + if v.Status.DeployedVersion == nil { + return false + } + return v.Spec.TargetVersion != *v.Status.DeployedVersion +} + +// validateMinorUpdateTargetStageAnnotationProgress rejects moving the target-stage +// annotation to an earlier rollout stage while a minor update is in progress, and rejects +// adding the annotation behind stages already completed when it was absent at update start. +func validateMinorUpdateTargetStageAnnotationProgress(old, updated *OpenStackVersion) error { + if !minorUpdateInProgress(updated) { + return nil + } + oldStage, oldOK := MinorUpdateTargetStageFromAnnotations(old.Annotations) + newStage, newOK := MinorUpdateTargetStageFromAnnotations(updated.Annotations) + if !newOK { + return nil + } + newIdx, okNew := MinorUpdateTargetStageIndex(newStage) + if !okNew { + return nil + } + annotationField := "metadata.annotations[" + MinorUpdateTargetStageAnnotation + "]" + gr := schema.GroupResource{ + Group: GroupVersion.WithKind("OpenStackVersion").Group, + Resource: GroupVersion.WithKind("OpenStackVersion").Kind, + } + + if !oldOK { + latest := LatestCompletedMinorUpdateTargetStageIndex(old.Status) + if latest >= 0 && newIdx < latest { + completedStage := validMinorUpdateTargetStagesOrdered[latest] + return apierrors.NewForbidden( + gr, updated.GetName(), &field.Error{ + Type: field.ErrorTypeForbidden, + Field: annotationField, + BadValue: newStage, + Detail: fmt.Sprintf( + "Cannot set update target stage to %q while minor update is in progress: update has already completed stage %q (targetVersion %q, deployedVersion %q); choose a further stage", + newStage, completedStage, updated.Spec.TargetVersion, *updated.Status.DeployedVersion, + ), + }, + ) + } + return nil + } + + oldIdx, _ := MinorUpdateTargetStageIndex(oldStage) + if newIdx >= oldIdx { + return nil + } + return apierrors.NewForbidden( + gr, updated.GetName(), &field.Error{ + Type: field.ErrorTypeForbidden, + Field: annotationField, + BadValue: newStage, + Detail: fmt.Sprintf( + "Cannot move update target stage from %q to earlier stage %q while minor update is in progress (targetVersion %q, deployedVersion %q); remove the annotation or set a further stage", + oldStage, newStage, updated.Spec.TargetVersion, *updated.Status.DeployedVersion, + ), + }, + ) +} + // hasAnyCustomImage checks if any image field in CustomContainerImages is set func hasAnyCustomImage(images CustomContainerImages) bool { // Check CinderVolumeImages map diff --git a/api/core/v1beta1/openstackversion_webhook_test.go b/api/core/v1beta1/openstackversion_webhook_test.go index 8c3e046a0..fb40c2ea1 100644 --- a/api/core/v1beta1/openstackversion_webhook_test.go +++ b/api/core/v1beta1/openstackversion_webhook_test.go @@ -2,6 +2,7 @@ package v1beta1 import ( "context" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -181,4 +182,297 @@ var _ = Describe("OpenStackVersion Webhook", func() { Expect(err.Error()).To(ContainSubstring("failed to convert old object to OpenStackVersion")) }) }) + + Context("MinorUpdateTargetStageAnnotation validation", func() { + + BeforeEach(func() { + SetupOpenStackVersionDefaults(OpenStackVersionDefaults{ + AvailableVersion: "1.1.0", + }) + }) + + It("should reject update when annotation value is invalid", func() { + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + newVersion.Annotations = map[string]string{ + MinorUpdateTargetStageAnnotation: "tyop", + } + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`Invalid target stage "tyop"`)) + Expect(err.Error()).To(ContainSubstring("Must be one of: " + MinorUpdateStageOVNControlplane)) + }) + + It("should reject update when annotation is present but empty", func() { + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + newVersion.Annotations = map[string]string{ + MinorUpdateTargetStageAnnotation: "", + } + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Annotation value must not be empty")) + }) + + It("should allow update when annotation is a valid stage", func() { + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + newVersion.Annotations = map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageRabbitMQ, + } + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should reject moving target stage backward during minor update", func() { + deployed := "1.0.0" + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + Annotations: map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageRabbitMQ, + }, + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + DeployedVersion: &deployed, + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.0.0": {}, + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + newVersion.Annotations[MinorUpdateTargetStageAnnotation] = MinorUpdateStageOVNDataplane + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Cannot move update target stage")) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf(`from %q to earlier stage %q while minor update is in progress`, MinorUpdateStageRabbitMQ, MinorUpdateStageOVNDataplane))) + }) + + It("should allow advancing target stage during minor update", func() { + deployed := "1.0.0" + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + Annotations: map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageOVNControlplane, + }, + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + DeployedVersion: &deployed, + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.0.0": {}, + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + newVersion.Annotations[MinorUpdateTargetStageAnnotation] = MinorUpdateStageOVNDataplane + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should allow removing target stage annotation during minor update", func() { + deployed := "1.0.0" + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + Annotations: map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageKeystone, + }, + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + DeployedVersion: &deployed, + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.0.0": {}, + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + delete(newVersion.Annotations, MinorUpdateTargetStageAnnotation) + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should allow moving target stage backward when minor update is not in progress for preparation to update", func() { + deployed := "1.1.0" + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + Annotations: map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageRabbitMQ, + }, + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + DeployedVersion: &deployed, + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.1.0": {}, + }, + }, + } + newVersion := oldVersion.DeepCopy() + newVersion.Annotations[MinorUpdateTargetStageAnnotation] = MinorUpdateStageOVNControlplane + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should reject adding target stage behind completed progress during minor update", func() { + deployed := "1.0.0" + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + DeployedVersion: &deployed, + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.0.0": {}, + "1.1.0": {}, + }, + }, + } + oldVersion.Status.Conditions.MarkTrue( + OpenStackVersionMinorUpdateOVNControlplane, + OpenStackVersionMinorUpdateReadyMessage, + ) + oldVersion.Status.Conditions.MarkTrue( + OpenStackVersionMinorUpdateOVNDataplane, + OpenStackVersionMinorUpdateReadyMessage, + ) + + newVersion := oldVersion.DeepCopy() + newVersion.Annotations = map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageOVNControlplane, + } + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Cannot set update target stage")) + Expect(err.Error()).To(ContainSubstring(MinorUpdateStageOVNControlplane)) + Expect(err.Error()).To(ContainSubstring(MinorUpdateStageOVNDataplane)) + }) + + It("should allow adding target stage at current progress during minor update", func() { + deployed := "1.0.0" + oldVersion := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + Status: OpenStackVersionStatus{ + DeployedVersion: &deployed, + ContainerImageVersionDefaults: map[string]*ContainerDefaults{ + "1.0.0": {}, + "1.1.0": {}, + }, + }, + } + oldVersion.Status.Conditions.MarkTrue( + OpenStackVersionMinorUpdateOVNControlplane, + OpenStackVersionMinorUpdateReadyMessage, + ) + + newVersion := oldVersion.DeepCopy() + newVersion.Annotations = map[string]string{ + MinorUpdateTargetStageAnnotation: MinorUpdateStageOVNControlplane, + } + + _, err := newVersion.ValidateUpdate(context.Background(), oldVersion, nil) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("ValidateCreate MinorUpdateTargetStageAnnotation validation", func() { + + BeforeEach(func() { + SetupOpenStackVersionDefaults(OpenStackVersionDefaults{ + AvailableVersion: "1.1.0", + }) + }) + + It("should reject create when annotation value is invalid", func() { + version := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + Annotations: map[string]string{ + MinorUpdateTargetStageAnnotation: "tyop", + }, + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + } + + _, err := version.ValidateCreate(context.Background(), nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`Invalid target stage "tyop"`)) + }) + + It("should reject create when annotation is present but empty", func() { + version := &OpenStackVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-version", + Namespace: "test-namespace", + Annotations: map[string]string{ + MinorUpdateTargetStageAnnotation: "", + }, + }, + Spec: OpenStackVersionSpec{TargetVersion: "1.1.0"}, + } + + _, err := version.ValidateCreate(context.Background(), nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Annotation value must not be empty")) + }) + }) }) diff --git a/docs/assemblies/proc_minor-update-staged-rollout.adoc b/docs/assemblies/proc_minor-update-staged-rollout.adoc new file mode 100644 index 000000000..92103f364 --- /dev/null +++ b/docs/assemblies/proc_minor-update-staged-rollout.adoc @@ -0,0 +1,337 @@ +[id="proc_minor-update-staged-rollout_{context}"] += Performing a staged update of OpenStack + +[role="_abstract"] + +A minor update of an OpenStack environment proceeds through a fixed sequence of stages. +By default the update runs all stages automatically. The +`core.openstack.org/update-target-stage` annotation on the `OpenStackVersion` CR lets you +pause the update after any stage so you can validate the environment, coordinate maintenance +windows, or advance one stage at a time. + +When a pause is active, the `OpenStackVersion` controller sets the next stage's condition to +`False` with a gated message, and the `OpenStackControlPlane` controller skips reconciling +control-plane components for stages beyond the annotation target until you advance or remove +the annotation. + +== Examples to use staged rollouts + +* You want to verify OVN networking is healthy before allowing the rest of the update to +proceed. +* Your organisation requires a sign-off after each major component is updated. +* You are performing the update in phases across a maintenance window and need to stop at a +known safe point. + +== Understanding the update pipeline + +The update always runs stages in this order. Each stage must complete before the next one +starts. + +[cols="2,3,2", options="header"] +|=== +| Stage | What gets updated | Requires manual action? + +| `ovn-controlplane` +| OVN control plane images +| No + +| `ovn-dataplane` +| OVN controller data plane images on compute nodes +| *Yes* — create an OVN `OpenStackDataPlaneDeployment` + +| `rabbitmq` +| RabbitMQ images +| No + +| `mariadb` +| MariaDB/Galera images +| No + +| `memcached` +| Memcached images +| No + +| `keystone` +| Keystone API images +| No + +| `controlplane` +| All remaining control-plane services +| No + +| _(completion)_ +| Data-plane services on compute nodes +| *Yes* — create a full `OpenStackDataPlaneDeployment` +|=== + +[NOTE] +Two stages require you to create an `OpenStackDataPlaneDeployment` manually. +The `ovn-dataplane` stage and the final data-plane completion step do not self-drive — +the controller waits for the corresponding deployment to finish before advancing. +See <>. + +== Prerequisites + +* A running cluster with a deployed OpenStack environment. +* `OpenStackControlPlane` and `OpenStackVersion` are both `Ready`. +* `status.deployedVersion` is set on the `OpenStackVersion` CR. +* A newer version is available: `status.availableVersion` differs from +`status.deployedVersion`. + +The examples below use: + +* Namespace: `openstack` +* `OpenStackVersion` CR name: `openstack` + +== Performing a fully staged update + +The recommended approach is to set the annotation to the first stage before bumping +`targetVersion`, then advance the annotation one stage at a time after you have validated +each step. If you start the update without the annotation, you cannot add it later at a +stage earlier than rollout progress already reached; set a later stage or remove the +annotation to run to completion. + +=== Step 1 — Confirm an update is available + +[source,bash,subs="+quotes"] +---- +$ oc get openstackversion openstack -n openstack \ + -o jsonpath='Available: {.status.availableVersion} Deployed: {.status.deployedVersion}{"\n"}' +---- + +Note the `availableVersion` value — this is `` in the commands below. + +=== Step 2 — Set the initial pause point + +Choose the stage after which you want the first pause. To pause after OVN control-plane: + +[source,bash] +---- +$ oc annotate openstackversion openstack \ + core.openstack.org/update-target-stage=ovn-controlplane \ + -n openstack +---- + +=== Step 3 — Start the update + +[source,bash,subs="+quotes"] +---- +$ oc patch openstackversion openstack -n openstack \ + --type=merge -p '{"spec":{"targetVersion":""}}' +---- + +The update begins immediately. The controller runs the `ovn-controlplane` stage and then +pauses. The `MinorUpdateOVNControlplane` condition becomes `True` and the +`MinorUpdateOVNDataplane` condition shows: + +---- +Minor update progression stopped after stage: ovn-controlplane. Set annotation to any stage after ovn-controlplane to resume OpenStack update or remove the annotation to run to completion. +---- + +=== Step 4 — Validate and advance stage by stage + +After each pause, check the environment is healthy, then advance to the next stage. + +==== Checking the current update status + +[source,bash,subs="+quotes"] +---- +$ oc get openstackversion openstack -n openstack \ + -o jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' \ + | grep MinorUpdate +---- + +Completed stages show `True`. The currently blocked stage shows `False` with a message +telling you which stage just finished and what to set next. + +==== Advancing to the next stage + +Update the annotation value to the stage you want to run next. For example, after +validating the OVN control-plane, advance to `ovn-dataplane`: + +[NOTE] +Before advancing to `ovn-dataplane`, create the OVN dataplane deployment first — +see <>. + +[source,bash] +---- +$ oc annotate openstackversion openstack \ + core.openstack.org/update-target-stage=ovn-dataplane \ + --overwrite -n openstack +---- + +Continue advancing through the remaining stages as needed: + +[cols="2,2", options="header"] +|=== +| To run through… | Set annotation to… + +| RabbitMQ +| `rabbitmq` + +| MariaDB +| `mariadb` + +| Memcached +| `memcached` + +| Keystone +| `keystone` + +| Full control-plane +| `controlplane` +|=== + +=== Step 5 — Complete the update + +When you are ready to run the final data-plane update on compute nodes, first create the +full dataplane deployment (see <>), +then remove the annotation to let the update finish: + +[source,bash] +---- +$ oc annotate openstackversion openstack \ + core.openstack.org/update-target-stage- \ + -n openstack +---- + +[NOTE] +The trailing `-` removes the annotation entirely. + +The controller runs the remaining stages and, once complete, sets +`status.deployedVersion` to the new version. + +=== Step 6 — Confirm completion + +[source,bash] +---- +$ oc get openstackversion openstack -n openstack \ + -o jsonpath='{.status.deployedVersion}' +---- + +The output should show ``. + +[[required-manual-deployments]] +== Required manual deployments + +Two stages in the process do not self-start. You must create an +`OpenStackDataPlaneDeployment` before (or at the same time as) advancing past each of them. + +=== OVN data-plane deployment + +Required before the `ovn-dataplane` stage can complete. This deployment updates only the +OVN-related services on compute nodes. + +[source,yaml] +---- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-deployment-ovn-update + namespace: openstack +spec: + nodeSets: + - openstack-edpm-ipam + servicesOverride: + - ovn +---- + +[source,bash] +---- +$ oc apply -f edpm-deployment-ovn-update.yaml +---- + +=== Full data-plane update deployment + +Required before the final completion step can finish. This deployment updates all remaining +services on compute nodes. + +[source,yaml] +---- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-deployment-update + namespace: openstack +spec: + nodeSets: + - openstack-edpm-ipam + servicesOverride: + - update +---- + +[source,bash] +---- +$ oc apply -f edpm-deployment-update.yaml +---- + +== Pausing a running update + +If you need to pause an update that is already in progress, add the annotation at any time. +The controller completes whichever stage is currently running, then stops after the stage you +named. + +[source,bash] +---- +$ oc annotate openstackversion openstack \ + core.openstack.org/update-target-stage= \ + -n openstack +---- + +Replace `` with the name of the last stage you want to run before pausing. + +[NOTE] +The webhook rejects setting the annotation to a stage that has already been +completed. If you need to pause, choose a stage at or ahead of the current progress. To +see which stages have completed, check the `MinorUpdate*` conditions on the +`OpenStackVersion` status. + +== Running the full update without pausing + +If you do not need staged control, omit the annotation entirely and let the controller run +all stages automatically. You still need to create both dataplane deployments at the right +time: + +. Create the OVN dataplane deployment before or immediately after starting the update. +. Create the dataplane update deployment before the final completion step. + +[source,bash,subs="+quotes"] +---- +$ oc patch openstackversion openstack -n openstack \ + --type=merge -p '{"spec":{"targetVersion":""}}' +---- + +== Troubleshooting + +=== The update appears stuck + +Check whether the blocked condition message contains `"stopped after stage"`. If it does, +the update is intentionally paused — advance or remove the annotation to continue. + +[source,bash] +---- +$ oc get openstackversion openstack -n openstack -o json | \ + jq '[.status.conditions[] | select(.reason=="Requested" and .status=="False")]' +---- + +=== `MinorUpdateOVNDataplane` or `MinorUpdateDataplane` stays `False` + +These stages wait for an `OpenStackDataPlaneDeployment` to complete. Check whether the +required deployment exists and is running: + +[source,bash] +---- +$ oc get openstackdataplanedeployment -n openstack +---- + +If the deployment is missing, create it as described in +<>. + +=== Checking overall update progress + +[source,bash,subs="+quotes"] +---- +$ watch -n 5 "oc get openstackversion openstack -n openstack \ + -o jsonpath='{range .status.conditions[*]}{.type}{\"\t\"}{.status}{\"\t\"}{.message}{\"\n\"}{end}' \ + | grep MinorUpdate" +---- diff --git a/docs/assemblies/updating-the-data-plane.adoc b/docs/assemblies/updating-the-data-plane.adoc index eed11ee3f..6e2a27af8 100644 --- a/docs/assemblies/updating-the-data-plane.adoc +++ b/docs/assemblies/updating-the-data-plane.adoc @@ -12,12 +12,18 @@ with a minor update of the control plane. OVN containers on the data plane nodes should not be updated until OVN containers on the control plane have been updated. +To pause the control-plane minor update after each stage—for validation use the +`core.openstack.org/update-target-stage` annotation on the `OpenStackVersion` CR. +See <>. + See https://github.com/openstack-k8s-operators/dev-docs/blob/main/version_updates.md[OpenStackVersion] and https://github.com/openstack-k8s-operators/dev-docs/blob/main/ovs-update.md[Open vSwitch update] for more information. +include::proc_minor-update-staged-rollout.adoc[leveloffset=+1] + include::proc_updating-the-data-plane-ovn.adoc[leveloffset=+1] include::proc_updating-the-data-plane.adoc[leveloffset=+1] diff --git a/internal/controller/core/openstackcontrolplane_controller.go b/internal/controller/core/openstackcontrolplane_controller.go index fca27f42b..01563767f 100644 --- a/internal/controller/core/openstackcontrolplane_controller.go +++ b/internal/controller/core/openstackcontrolplane_controller.go @@ -314,12 +314,13 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, nil } - // OVN + // OVN control-plane stage (respect update-target-stage on OpenStackVersion) // Once the OVN controlplane phase is complete, skip reconcileOVNControllers to prevent // transient OVN readiness changes from flapping OVNReadyCondition during // later phases. We still need to mark OVNReadyCondition True because // InitConditions() resets all conditions to Unknown on each reconcile. - if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane) { + if corev1beta1.MinorUpdateStageAllowedForReconcile(version.Annotations, corev1beta1.MinorUpdateStageOVNControlplane) && + !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane) { Log.Info("Minor update OVN on the ControlPlane") ctrlResult, err = r.reconcileOVNControllers(ctx, instance, version, helper) if err != nil { @@ -331,65 +332,70 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr // Wait for the OpenStackVersion controller to acknowledge OVN controlplane update return ctrlResult, nil } + if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNControlplane) { + return ctrl.Result{}, nil + } instance.Status.Conditions.MarkTrue(corev1beta1.OpenStackControlPlaneOVNReadyCondition, corev1beta1.OpenStackControlPlaneOVNReadyMessage) + if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNDataplane) { + return ctrl.Result{}, nil + } // only if OVN dataplane is updated - if version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateOVNDataplane) { - Log.Info("Minor update in progress") - - // RabbitMQ + Log.Info("Minor update OVN dataplane completed.") + // RabbitMQ + if corev1beta1.MinorUpdateStageAllowedForReconcile(version.Annotations, corev1beta1.MinorUpdateStageRabbitMQ) { ctrlResult, err = openstack.ReconcileRabbitMQs(ctx, instance, version, helper) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil - } else { - if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateRabbitMQ) { - Log.Info("Returning for RabbitMQ minor update reconcile") - return ctrlResult, nil - } } - - // Galara + if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateRabbitMQ) { + Log.Info("Returning for RabbitMQ minor update reconcile") + return ctrl.Result{}, nil + } + } + // Galara + if corev1beta1.MinorUpdateStageAllowedForReconcile(version.Annotations, corev1beta1.MinorUpdateStageMariaDB) { ctrlResult, err = openstack.ReconcileGaleras(ctx, instance, version, helper) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil - } else { - if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateMariaDB) { - Log.Info("Returning for Galara minor update reconcile") - return ctrlResult, nil - } } - - // Memcached + if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateMariaDB) { + Log.Info("Returning for Galara minor update reconcile") + return ctrl.Result{}, nil + } + } + // Memcached + if corev1beta1.MinorUpdateStageAllowedForReconcile(version.Annotations, corev1beta1.MinorUpdateStageMemcached) { ctrlResult, err = openstack.ReconcileMemcacheds(ctx, instance, version, helper) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil - } else { - if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateMemcached) { - Log.Info("Returning for Memcached minor update reconcile") - return ctrlResult, nil - } } - - // Keystone API + if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateMemcached) { + Log.Info("Returning for Memcached minor update reconcile") + return ctrl.Result{}, nil + } + } + // Keystone API + if corev1beta1.MinorUpdateStageAllowedForReconcile(version.Annotations, corev1beta1.MinorUpdateStageKeystone) { ctrlResult, err = openstack.ReconcileKeystoneAPI(ctx, instance, version, helper) if err != nil { return ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { return ctrlResult, nil - } else { - if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateKeystone) { - Log.Info("Returning for KeystoneAPI minor update reconcile") - return ctrlResult, nil - } } - - // the rest of the controlplane + if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateKeystone) { + Log.Info("Returning for KeystoneAPI minor update reconcile") + return ctrl.Result{}, nil + } + } + // the rest of the controlplane + if corev1beta1.MinorUpdateStageAllowedForReconcile(version.Annotations, corev1beta1.MinorUpdateStageControlplane) { ctrlResult, err = r.reconcileNormal(ctx, instance, version, helper) if err != nil { return ctrl.Result{}, err @@ -400,7 +406,7 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr instance.Status.DeployedVersion = &version.Spec.TargetVersion if !version.Status.Conditions.IsTrue(corev1beta1.OpenStackVersionMinorUpdateControlplane) { Log.Info("Returning for ControlPlane minor update reconcile") - return ctrlResult, nil + return ctrl.Result{}, nil } } } diff --git a/internal/controller/core/openstackversion_controller.go b/internal/controller/core/openstackversion_controller.go index 9f0f0d40a..4b461a4fd 100644 --- a/internal/controller/core/openstackversion_controller.go +++ b/internal/controller/core/openstackversion_controller.go @@ -262,6 +262,34 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req // minor update in progress if instance.Status.DeployedVersion != nil && instance.Spec.TargetVersion != *instance.Status.DeployedVersion { + // targetStage is the value of the target-stage annotation. When set, the update + // completes all stages up to and including the named stage, then pauses. An empty + // string (annotation absent or invalid) means run to completion without pausing. + targetStage := "" + if stage, ok := corev1beta1.MinorUpdateTargetStageFromAnnotations(instance.Annotations); ok { + targetStage = stage + } + + // gateNextStage marks the next condition as blocked and returns whether + // the gate was applied. Callers should only exit reconcile when gated == true. + gateNextStage := func(completedStage string, nextCondition condition.Type) (ctrl.Result, bool) { + if !instance.Status.Conditions.IsTrue(nextCondition) { + instance.Status.Conditions.Set(condition.FalseCondition( + nextCondition, + corev1beta1.OpenStackVersionMinorUpdateGatedReason, + condition.SeverityInfo, + corev1beta1.OpenStackVersionMinorUpdateReadyGatedMessage, + completedStage, completedStage)) + Log.Info("Minor update paused at target stage", "stage", completedStage, + "annotation", corev1beta1.MinorUpdateTargetStageAnnotation) + return ctrl.Result{}, true + } + Log.Info("Skipping gate for stage already completed", "completedStage", completedStage, + "nextCondition", nextCondition, + "annotation", corev1beta1.MinorUpdateTargetStageAnnotation) + return ctrl.Result{}, false + } + // Only check OVN when enabled to avoid hanging on a removed condition if controlPlane.Spec.Ovn.Enabled { if !openstack.OVNControllerImageMatch(ctx, controlPlane, instance) || @@ -279,6 +307,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateOVNControlplane, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageOVNControlplane { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageOVNControlplane, + corev1beta1.OpenStackVersionMinorUpdateOVNDataplane); gated { + return result, nil + } + } + // minor update for Dataplane OVN // Only check OVN when enabled to avoid hanging on a removed condition if controlPlane.Spec.Ovn.Enabled { @@ -307,8 +342,10 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req // - If no running OVN deployment AND the previous condition was // False/RequestedReason: the deployment we saw previously has // completed → proceed (fall through to set True) + // - If the previous reason was Gated (target-stage pause), treat + // like Init — not evidence that a deployment completed // - If no running OVN deployment AND the previous condition was - // NOT False/RequestedReason (e.g. still Unknown from Init): + // NOT False/RequestedReason (e.g. still Unknown from Init or Gated): // we haven't seen a deployment yet → keep waiting // // When the image differs between versions, the image match alone @@ -365,6 +402,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateOVNDataplane, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageOVNDataplane { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageOVNDataplane, + corev1beta1.OpenStackVersionMinorUpdateRabbitMQ); gated { + return result, nil + } + } + // minor update for RabbitMQ if !openstack.RabbitmqImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneRabbitMQReadyCondition) { @@ -380,6 +424,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateRabbitMQ, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageRabbitMQ { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageRabbitMQ, + corev1beta1.OpenStackVersionMinorUpdateMariaDB); gated { + return result, nil + } + } + // minor update for MariaDB if !openstack.GaleraImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMariaDBReadyCondition) { @@ -395,6 +446,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateMariaDB, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageMariaDB { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageMariaDB, + corev1beta1.OpenStackVersionMinorUpdateMemcached); gated { + return result, nil + } + } + // minor update for Memcached if !openstack.MemcachedImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMemcachedReadyCondition) { @@ -410,6 +468,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateMemcached, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageMemcached { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageMemcached, + corev1beta1.OpenStackVersionMinorUpdateKeystone); gated { + return result, nil + } + } + // minor update for Keystone API if !openstack.KeystoneImageMatch(ctx, controlPlane, instance) || !controlPlane.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneKeystoneAPIReadyCondition) { @@ -425,6 +490,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateKeystone, corev1beta1.OpenStackVersionMinorUpdateReadyMessage) + if targetStage == corev1beta1.MinorUpdateStageKeystone { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageKeystone, + corev1beta1.OpenStackVersionMinorUpdateControlplane); gated { + return result, nil + } + } + // minor update for Controlplane in progress if !controlPlane.IsReady() { instance.Status.Conditions.Set(condition.FalseCondition( @@ -456,6 +528,13 @@ func (r *OpenStackVersionReconciler) Reconcile(ctx context.Context, req ctrl.Req corev1beta1.OpenStackVersionMinorUpdateReadyMessage) Log.Info("Minor update for ControlPlane completed") + if targetStage == corev1beta1.MinorUpdateStageControlplane { + if result, gated := gateNextStage(corev1beta1.MinorUpdateStageControlplane, + corev1beta1.OpenStackVersionMinorUpdateDataplane); gated { + return result, nil + } + } + if !openstack.DataplaneNodesetsDeployed(instance, dataplaneNodesets) { instance.Status.Conditions.Set(condition.FalseCondition( corev1beta1.OpenStackVersionMinorUpdateDataplane, diff --git a/test/functional/ctlplane/openstackversion_controller_test.go b/test/functional/ctlplane/openstackversion_controller_test.go index 1438506d3..931b21de3 100644 --- a/test/functional/ctlplane/openstackversion_controller_test.go +++ b/test/functional/ctlplane/openstackversion_controller_test.go @@ -35,7 +35,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -var _ = Describe("OpenStackOperator controller", func() { +var _ = Describe("OpenStackVersion controller", func() { BeforeEach(func() { // lib-common uses OPERATOR_TEMPLATES env var to locate the "templates" // directory of the operator. We need to set them othervise lib-common @@ -1585,4 +1585,367 @@ var _ = Describe("OpenStackOperator controller", func() { }) }) + // Test target-stage annotation gates minor update at the specified stage + When("Minor update with target-stage annotation", func() { + var ( + initialVersion = "old" + updatedVersion = "0.0.1" + testRabbitMQImage = "foo/rabbit:0.0.2" + testMariaDBImage = "foo/maria:0.0.2" + testMemcachedImage = "foo/memcached:0.0.2" + testKeystoneAPIImage = "foo/keystone:0.0.2" + ) + + BeforeEach(func() { + // Lightweight controlplane spec with OVN DISABLED so that the OVN stages + // auto-complete, making it straightforward to gate at "ovn-controlplane" + // without needing to simulate OVN readiness in these tests. + spec := GetDefaultOpenStackControlPlaneSpec() + + galeraTemplate := map[string]interface{}{ + names.DBName.Name: map[string]interface{}{ + "storageRequest": "500M", + }, + } + spec["galera"] = map[string]interface{}{ + "enabled": true, + "templates": galeraTemplate, + } + + spec["horizon"] = map[string]interface{}{"enabled": false} + spec["glance"] = map[string]interface{}{"enabled": false} + spec["cinder"] = map[string]interface{}{"enabled": false} + spec["neutron"] = map[string]interface{}{"enabled": false} + spec["manila"] = map[string]interface{}{"enabled": false} + spec["heat"] = map[string]interface{}{"enabled": false} + spec["telemetry"] = map[string]interface{}{"enabled": false} + spec["tls"] = GetTLSPublicSpec() + + // OVN disabled — stages auto-complete during minor update + spec["ovn"] = map[string]interface{}{ + "enabled": false, + } + + DeferCleanup( + th.DeleteInstance, + CreateOpenStackVersion(names.OpenStackVersionName, GetDefaultOpenStackVersionSpec()), + ) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) + + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCAPublicName)) + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCAInternalName)) + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCAOvnName)) + DeferCleanup(k8sClient.Delete, ctx, CreateCertSecret(names.RootCALibvirtName)) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.DBCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.DBCell1CertName)) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) + + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionInitialized, + k8s_corev1.ConditionTrue, + ) + + version := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(version).Should(Not(BeNil())) + + g.Expect(*version.Status.AvailableVersion).Should(ContainSubstring("0.0.1")) + g.Expect(version.Spec.TargetVersion).Should(ContainSubstring("0.0.1")) + updatedVersion = *version.Status.AvailableVersion + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Status.ContainerImageVersionDefaults[initialVersion] = version.Status.ContainerImageVersionDefaults[updatedVersion] + version.Status.ContainerImageVersionDefaults[initialVersion].RabbitmqImage = &testRabbitMQImage + version.Status.ContainerImageVersionDefaults[initialVersion].MariadbImage = &testMariaDBImage + version.Status.ContainerImageVersionDefaults[initialVersion].InfraMemcachedImage = &testMemcachedImage + version.Status.ContainerImageVersionDefaults[initialVersion].KeystoneAPIImage = &testKeystoneAPIImage + g.Expect(th.K8sClient.Status().Update(th.Ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Spec.TargetVersion = initialVersion + g.Expect(th.K8sClient.Update(th.Ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(osversion).Should(Not(BeNil())) + g.Expect(osversion.Generation).Should(Equal(osversion.Status.ObservedGeneration)) + + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionInitialized, + k8s_corev1.ConditionTrue, + ) + + g.Expect(*osversion.Status.AvailableVersion).Should(Equal(updatedVersion)) + g.Expect(osversion.Spec.TargetVersion).Should(Equal(initialVersion)) + g.Expect(osversion.Status.DeployedVersion).Should(BeNil()) + }, timeout, interval).Should(Succeed()) + + DeferCleanup( + th.DeleteInstance, + CreateOpenStackControlPlane(names.OpenStackControlplaneName, spec), + ) + + DeferCleanup( + th.DeleteInstance, + CreateDataplaneNodeSet(names.OpenStackVersionName, DefaultDataPlaneNoNodeSetSpec(false)), + ) + + dataplanenodeset := GetDataplaneNodeset(names.OpenStackVersionName) + dataplanenodeset.Status.DeployedVersion = initialVersion + Expect(th.K8sClient.Status().Update(th.Ctx, dataplanenodeset)).To(Succeed()) + + th.CreateSecret(types.NamespacedName{Name: "openstack-config-secret", Namespace: namespace}, map[string][]byte{"secure.yaml": []byte("foo")}) + th.CreateConfigMap(types.NamespacedName{Name: "openstack-config", Namespace: namespace}, map[string]interface{}{"clouds.yaml": string("foo"), "OS_CLOUD": "default"}) + + OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) + Expect(OSCtlplane.Spec.Ovn.Enabled).Should(BeFalse()) + + SimulateControlplaneReady() + + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackControlplaneName, + ConditionGetterFunc(OpenStackControlPlaneConditionGetter), + condition.ReadyCondition, + k8s_corev1.ConditionTrue, + ) + OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) + g.Expect(OSCtlplane.Status.DeployedVersion).Should(Equal(&initialVersion)) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(osversion).Should(Not(BeNil())) + g.Expect(osversion.Generation).Should(Equal(osversion.Status.ObservedGeneration)) + g.Expect(osversion.Status.DeployedVersion).Should(Equal(&initialVersion)) + }, timeout, interval).Should(Succeed()) + }) + + It("should complete the named stage then block the next stage", Serial, func() { + // Set target-stage to "ovn-controlplane". With OVN disabled the OVN + // controlplane stage auto-completes (MarkTrue); the controller then + // detects the gate and blocks the OVN dataplane stage. + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Trigger minor update + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for initialization + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + g.Expect(osversion).Should(Not(BeNil())) + g.Expect(osversion.Generation).Should(Equal(osversion.Status.ObservedGeneration)) + + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionInitialized, + k8s_corev1.ConditionTrue, + ) + }, timeout, interval).Should(Succeed()) + + // OVN controlplane stage must have completed (True) + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNControlplane, + k8s_corev1.ConditionTrue, + ) + + // OVN dataplane stage must be gated (False with target-stage message) + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Status).Should(Equal(k8s_corev1.ConditionFalse)) + g.Expect(cond.Reason).Should(Equal(condition.Reason(corev1.OpenStackVersionMinorUpdateGatedReason))) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // DeployedVersion must not advance while gated + osversion := GetOpenStackVersion(names.OpenStackVersionName) + Expect(osversion.Status.DeployedVersion).Should(Equal(&initialVersion)) + }) + + It("should resume update when the annotation is removed", Serial, func() { + // Gate at "ovn-controlplane" + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Trigger minor update + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for gated state: OVNDataplane blocked + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // Remove the annotation to let the update proceed + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + delete(version.Annotations, corev1.MinorUpdateTargetStageAnnotation) + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // OVN dataplane should no longer be gated; with OVN disabled it + // auto-completes so the condition becomes True. + Eventually(func(_ Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNDataplane, + k8s_corev1.ConditionTrue, + ) + }, timeout, interval).Should(Succeed()) + }) + + It("should advance gate to a later stage when annotation value is updated", Serial, func() { + // Start gated at "ovn-controlplane" + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for OVNDataplane to be gated + Eventually(func(g Gomega) { + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // Advance gate to "ovn-dataplane" to let OVN dataplane auto-complete + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNDataplane + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // OVN dataplane auto-completes (OVN disabled); RabbitMQ becomes gated + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNDataplane, + k8s_corev1.ConditionTrue, + ) + + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateRabbitMQ) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Status).Should(Equal(k8s_corev1.ConditionFalse)) + g.Expect(cond.Reason).Should(Equal(condition.Reason(corev1.OpenStackVersionMinorUpdateGatedReason))) + g.Expect(cond.Message).Should(ContainSubstring(corev1.MinorUpdateStageOVNDataplane)) + }, timeout, interval).Should(Succeed()) + }) + + It("should not let gate to a previous stage when annotation value is not set from beginning of the update", Serial, func() { + // Start minor update without target-stage annotation + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations != nil { + delete(version.Annotations, corev1.MinorUpdateTargetStageAnnotation) + } + version.Spec.TargetVersion = updatedVersion + g.Expect(k8sClient.Update(ctx, version)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(_ Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNDataplane, + k8s_corev1.ConditionTrue, + ) + }, timeout, interval).Should(Succeed()) + + SimulateRabbitmqReady() + + Eventually(func(_ Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateRabbitMQ, + k8s_corev1.ConditionTrue, + ) + }, timeout*4, interval).Should(Succeed()) + + // Retroactively adding an earlier pause point must be rejected by the webhook + Eventually(func(g Gomega) { + version := GetOpenStackVersion(names.OpenStackVersionName) + if version.Annotations == nil { + version.Annotations = make(map[string]string) + } + version.Annotations[corev1.MinorUpdateTargetStageAnnotation] = corev1.MinorUpdateStageOVNControlplane + err := k8sClient.Update(ctx, version) + g.Expect(err).Should(HaveOccurred()) + g.Expect(err.Error()).Should(ContainSubstring("Cannot set update target stage")) + g.Expect(err.Error()).Should(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + + // Controller must not re-gate stages that already completed + Eventually(func(g Gomega) { + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateOVNDataplane, + k8s_corev1.ConditionTrue, + ) + th.ExpectCondition( + names.OpenStackVersionName, + ConditionGetterFunc(OpenStackVersionConditionGetter), + corev1.OpenStackVersionMinorUpdateRabbitMQ, + k8s_corev1.ConditionTrue, + ) + + osversion := GetOpenStackVersion(names.OpenStackVersionName) + cond := osversion.Status.Conditions.Get(corev1.OpenStackVersionMinorUpdateOVNDataplane) + g.Expect(cond).ShouldNot(BeNil()) + g.Expect(cond.Message).ShouldNot(ContainSubstring(corev1.MinorUpdateStageOVNControlplane)) + }, timeout, interval).Should(Succeed()) + }) + }) + })