From 828c5cd4c996089c16677154fd02e0ad3024c010 Mon Sep 17 00:00:00 2001 From: Daniel Lawton Date: Fri, 4 Sep 2026 14:14:06 +0100 Subject: [PATCH] Add Availability Zones (AZ) scenario support Port the Nova + Cinder Availability Zones scenario from the legacy openshift-ir-plugin / rhos-infrared automation into the shiftstack-qa stages. The AZ job installs OCP IPI on OpenStack with one Nova aggregate per compute host (one AZ each), OCP masters/workers distributed evenly across the AZs, and rootVolumes placed on per-AZ Cinder backends. Post- install verification asserts the OCP nodes and their root volumes landed in the expected Nova and Cinder AZs. Stages/roles: - prepare: configure_az.yml creates host aggregates + per-AZ volume types, configure_az_ceph_crush.yml pins each per-AZ pool to its host-local OSDs. - install: discovers Nova/Cinder AZs (tools_cluster_checks/discover_az.yml) and renders zones + per-AZ rootVolume types/zones into install-config. - verification: check_azs.yml verifies Nova and Cinder AZ placement. - day2ops: scaleup_worker and cinder_registry (AZ-aware registry storage). - cleanup: remove_az.yml tears down aggregates and AZ volume types, gated by az_remove_on_cleanup. - jobs_definitions/availability_zones_4.22_nightly.yaml wires it together. All AZ behaviour is gated behind az_enable / openshift_root_volumes (default false), so non-AZ jobs are unaffected. Co-Authored-By: Claude Opus 4.8 Change-Id: I45bdf61b3bc8e270c950e2e07590a736a9cd81ce Signed-off-by: Daniel Lawton --- .../stages/roles/cleanup/defaults/main.yml | 3 + .../stages/roles/cleanup/tasks/main.yml | 4 + .../stages/roles/cleanup/tasks/remove_az.yml | 46 ++++ .../stages/roles/day2ops/defaults/main.yml | 1 + .../tasks/procedures/cinder_registry.yml | 257 ++++++++++++++++++ .../tasks/procedures/scaleup_worker.yml | 52 ++++ .../stages/roles/install/tasks/main.yml | 6 + .../templates/install-config-ipi.yaml.j2 | 34 ++- .../stages/roles/prepare/defaults/main.yml | 14 + .../roles/prepare/tasks/configure_az.yml | 82 ++++++ .../prepare/tasks/configure_az_ceph_crush.yml | 50 ++++ .../stages/roles/prepare/tasks/main.yml | 4 + .../stages/roles/prepare/tasks/project.yml | 2 +- .../roles/verification/tasks/check_azs.yml | 143 ++++++++++ .../stages/roles/verification/tasks/main.yml | 6 + .../availability_zones_4.22_nightly.yaml | 76 ++++++ 16 files changed, 778 insertions(+), 2 deletions(-) create mode 100644 collection/stages/roles/cleanup/tasks/remove_az.yml create mode 100644 collection/stages/roles/day2ops/tasks/procedures/cinder_registry.yml create mode 100644 collection/stages/roles/day2ops/tasks/procedures/scaleup_worker.yml create mode 100644 collection/stages/roles/prepare/tasks/configure_az.yml create mode 100644 collection/stages/roles/prepare/tasks/configure_az_ceph_crush.yml create mode 100644 collection/stages/roles/verification/tasks/check_azs.yml create mode 100644 jobs_definitions/availability_zones_4.22_nightly.yaml diff --git a/collection/stages/roles/cleanup/defaults/main.yml b/collection/stages/roles/cleanup/defaults/main.yml index 6cb48c1a..09e1c322 100644 --- a/collection/stages/roles/cleanup/defaults/main.yml +++ b/collection/stages/roles/cleanup/defaults/main.yml @@ -4,3 +4,6 @@ # Set the OpenShift installation type to None existing_ocp_installation_type: "" force_cleanup: false +az_remove_on_cleanup: false +az_volume_type_prefix: "fastpool-" +az_volume_type_count: 3 diff --git a/collection/stages/roles/cleanup/tasks/main.yml b/collection/stages/roles/cleanup/tasks/main.yml index 6a21a643..25ee1f42 100644 --- a/collection/stages/roles/cleanup/tasks/main.yml +++ b/collection/stages/roles/cleanup/tasks/main.yml @@ -68,6 +68,10 @@ - name: Discover the OpenShift installation type ansible.builtin.include_tasks: detect_ocp_installation.yml + - name: Remove availability zone configuration from OpenStack + ansible.builtin.include_tasks: remove_az.yml + when: az_remove_on_cleanup | default(false) | bool + - name: Cleanup IPv6 secondary network resources if they exist ansible.builtin.include_tasks: cleanup_ipv6_secondary.yml when: ocp_deployment_topology.secondary_ip_protocol | default('') == 'ipv6' diff --git a/collection/stages/roles/cleanup/tasks/remove_az.yml b/collection/stages/roles/cleanup/tasks/remove_az.yml new file mode 100644 index 00000000..5eb471fe --- /dev/null +++ b/collection/stages/roles/cleanup/tasks/remove_az.yml @@ -0,0 +1,46 @@ +--- +- name: Discover existing host aggregates + ansible.builtin.command: + cmd: openstack aggregate list -c Name -f value + environment: + OS_CLOUD: "{{ admin_cloud }}" + register: az_existing_aggregates + changed_when: false + +- name: Remove hosts from existing aggregates + openstack.cloud.host_aggregate: + cloud: "{{ admin_cloud }}" + state: present + name: "{{ item }}" + hosts: [] + loop: "{{ az_existing_aggregates.stdout_lines }}" + when: az_existing_aggregates.stdout_lines | length > 0 + +- name: Delete existing host aggregates + openstack.cloud.host_aggregate: + cloud: "{{ admin_cloud }}" + state: absent + name: "{{ item }}" + loop: "{{ az_existing_aggregates.stdout_lines }}" + when: az_existing_aggregates.stdout_lines | length > 0 + +- name: Show nova host AZ mapping after aggregate cleanup + ansible.builtin.command: + cmd: openstack host list -c Zone -c "Host Name" -f table + environment: + OS_CLOUD: "{{ admin_cloud }}" + register: az_host_list_after_cleanup + changed_when: false + +- name: Print nova host AZ mapping after aggregate cleanup + ansible.builtin.debug: + var: az_host_list_after_cleanup.stdout_lines + +- name: Delete AZ volume types + ansible.builtin.command: + cmd: openstack volume type delete {{ az_volume_type_prefix }}{{ item }} + environment: + OS_CLOUD: "{{ admin_cloud }}" + loop: "{{ range(0, az_volume_type_count | int) | list }}" + failed_when: false + changed_when: true diff --git a/collection/stages/roles/day2ops/defaults/main.yml b/collection/stages/roles/day2ops/defaults/main.yml index 4f6557f7..0486c82c 100644 --- a/collection/stages/roles/day2ops/defaults/main.yml +++ b/collection/stages/roles/day2ops/defaults/main.yml @@ -2,6 +2,7 @@ # defaults file for day2ops day2ops_steps: [] day2ops_report_filename: shiftstack-qa-day2ops-results.xml +registry_test_project: registry-test # Application Credentials rotation app_credential_name: "AppCreds-{{ user_cloud }}" diff --git a/collection/stages/roles/day2ops/tasks/procedures/cinder_registry.yml b/collection/stages/roles/day2ops/tasks/procedures/cinder_registry.yml new file mode 100644 index 00000000..6cb2ee41 --- /dev/null +++ b/collection/stages/roles/day2ops/tasks/procedures/cinder_registry.yml @@ -0,0 +1,257 @@ +--- +- name: Get existing workers + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: v1 + kind: Node + label_selectors: + - node-role.kubernetes.io/worker + register: workers + +- name: Get first worker name and its Cinder zone label + ansible.builtin.set_fact: + worker_name: "{{ workers.resources[0].metadata.labels['kubernetes.io/hostname'] }}" + nova_az_for_registry: "{{ workers.resources[0].metadata.labels['topology.cinder.csi.openstack.org/zone'] }}" + +- name: Discover Cinder AZ for the worker root volume + block: + - name: Query worker root volume AZ + ansible.builtin.shell: | + set -o pipefail + openstack volume show \ + "$(openstack volume list -c ID -c Name -f value | grep "{{ worker_name }}" | cut -d' ' -f2)" \ + -c availability_zone -f value + environment: + OS_CLOUD: "{{ user_cloud }}" + register: registry_worker_volume_az + failed_when: false + changed_when: false + + - name: Set Cinder AZ for registry PVC + ansible.builtin.set_fact: + cinder_az_for_registry: "{{ (registry_worker_volume_az.stdout | trim == '') | ternary('nova', registry_worker_volume_az.stdout | trim) }}" + +- name: Remove existing topology-aware storage class + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: absent + api_version: storage.k8s.io/v1 + kind: StorageClass + name: topology-aware-image-registry + +- name: Create a storage class for integrated registry + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + definition: + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: topology-aware-image-registry + provisioner: cinder.csi.openstack.org + parameters: + availability: "{{ cinder_az_for_registry }}" + volumeBindingMode: WaitForFirstConsumer + +- name: Create PVC for integrated registry + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + definition: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pvc-registry + namespace: openshift-image-registry + annotations: + imageregistry.openshift.io: "true" + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: topology-aware-image-registry + +- name: Trigger registry reconfiguration + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + definition: + apiVersion: imageregistry.operator.openshift.io/v1 + kind: Config + metadata: + name: cluster + spec: + defaultRoute: true + disableRedirect: false + rolloutStrategy: Recreate + replicas: 1 + storage: + managementState: Managed + pvc: + claim: pvc-registry + swift: null + status: null + state: present + +- name: Wait for registry pod on Running phase + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + kind: Pod + namespace: openshift-image-registry + label_selectors: + - docker-registry=default + register: registry_pod + retries: 60 + delay: 10 + until: + - registry_pod.resources[0].spec.volumes[0].persistentVolumeClaim is defined + - registry_pod.resources[0].spec.volumes[0].persistentVolumeClaim.claimName == "pvc-registry" + - registry_pod.resources[0].status is defined + - registry_pod.resources[0].status.phase == "Running" + +- name: Get integrated registry route + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: route.openshift.io/v1 + kind: Route + namespace: openshift-image-registry + name: default-route + register: registry_routes + +- name: Set integrated registry URL + ansible.builtin.set_fact: + registry_url: "{{ registry_routes.resources[0].spec.host }}" + +- name: Add registry domain resolution to /etc/hosts + block: + - name: Resolve apps ingress IP from /etc/hosts + ansible.builtin.shell: | + set -o pipefail + awk '{if ($2 ~ "apps.{{ ocp_cluster_name }}") print $1}' /etc/hosts | uniq + register: hosts_apps_ip + changed_when: false + + - name: Set apps ingress IP + ansible.builtin.set_fact: + apps_fip_ip: "{{ hosts_apps_ip.stdout }}" + + - name: Add registry floating IP to /etc/hosts + ansible.builtin.lineinfile: + path: /etc/hosts + line: "{{ apps_fip_ip }} {{ registry_url }}" + become: true + +- name: Delete the registry test project if it exists + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: absent + api_version: project.openshift.io/v1 + kind: Project + name: "{{ registry_test_project }}" + wait: true + wait_timeout: 60 + +- name: Create the registry test project + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + api_version: project.openshift.io/v1 + kind: Project + name: "{{ registry_test_project }}" + +- name: Add registry permissions to admin user + ansible.builtin.shell: | + set -o pipefail + oc project {{ registry_test_project }} + oc policy add-role-to-user registry-viewer admin + oc policy add-role-to-user registry-editor admin + environment: + KUBECONFIG: "{{ kubeconfig }}" + changed_when: true + +- name: Get builder service account token for OCP before 4.11 + when: openshift_release_build_name is version('4.11', '<') + block: + - name: Get builder token + ansible.builtin.command: oc serviceaccounts get-token builder + environment: + KUBECONFIG: "{{ kubeconfig }}" + register: registry_token_result + retries: 5 + delay: 60 + until: registry_token_result is succeeded + changed_when: false + + - name: Set registry token + ansible.builtin.set_fact: + registry_token: "{{ registry_token_result.stdout }}" + +- name: Get builder service account token for OCP 4.11 and later + when: openshift_release_build_name is version('4.11', '>=') + block: + - name: Create builder token + ansible.builtin.command: oc create token builder + environment: + KUBECONFIG: "{{ kubeconfig }}" + register: registry_token_result + retries: 5 + delay: 60 + until: registry_token_result is succeeded + changed_when: false + + - name: Set registry token + ansible.builtin.set_fact: + registry_token: "{{ registry_token_result.stdout }}" + +- name: Push demo image to integrated registry + ansible.builtin.shell: | + set -o pipefail + sudo podman login -u admin -p {{ registry_token }} --tls-verify=false {{ registry_url }} && \ + sudo podman pull quay.io/kuryr/demo && \ + sudo podman tag quay.io/kuryr/demo {{ registry_url }}/{{ registry_test_project }}/demo && \ + sudo podman push {{ registry_url }}/{{ registry_test_project }}/demo --tls-verify=false + register: registry_push_result + retries: 10 + delay: 30 + until: not registry_push_result.failed + changed_when: true + +- name: Create a pod using integrated registry + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: demo + namespace: "{{ registry_test_project }}" + spec: + replicas: 1 + selector: + matchLabels: + app: demo + template: + metadata: + labels: + app: demo + spec: + containers: + - name: demo + image: image-registry.openshift-image-registry.svc:5000/{{ registry_test_project }}/demo + ports: + - containerPort: 80 + +- name: Wait for demo pod to reach Running phase + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + kind: Pod + label_selectors: + - app=demo + register: demo_pod + retries: 5 + delay: 10 + until: + - demo_pod.resources[0].status is defined + - demo_pod.resources[0].status.phase == "Running" diff --git a/collection/stages/roles/day2ops/tasks/procedures/scaleup_worker.yml b/collection/stages/roles/day2ops/tasks/procedures/scaleup_worker.yml new file mode 100644 index 00000000..cb3d5974 --- /dev/null +++ b/collection/stages/roles/day2ops/tasks/procedures/scaleup_worker.yml @@ -0,0 +1,52 @@ +--- +- name: Get the worker machineset + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: machine.openshift.io/v1beta1 + kind: MachineSet + namespace: openshift-machine-api + label_selectors: + - machine.openshift.io/cluster-api-machine-type=worker + register: ms_worker + +- name: Set the expected workers replicas + ansible.builtin.set_fact: + worker_spec_replicas: "{{ ms_worker.resources[0].spec.replicas }}" + +- name: Check that the number of worker replicas are as expected + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: machine.openshift.io/v1beta1 + kind: MachineSet + namespace: openshift-machine-api + name: "{{ ms_worker.resources[0].metadata.name }}" + register: workers + until: workers | json_query('resources[0].status.readyReplicas') == worker_spec_replicas | int + retries: 40 + delay: 30 + +- name: Set the number of new workers + ansible.builtin.set_fact: + new_num_workers: "{{ ms_worker.resources[0].spec.replicas | int + 1 }}" + +- name: Scale workers + ansible.builtin.command: + cmd: >- + oc scale --replicas={{ new_num_workers }} + machineset/{{ ms_worker.resources[0].metadata.name }} + -n openshift-machine-api + environment: + KUBECONFIG: "{{ kubeconfig }}" + changed_when: true + +- name: Wait for all the workers to be up + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: machine.openshift.io/v1beta1 + kind: MachineSet + namespace: openshift-machine-api + name: "{{ ms_worker.resources[0].metadata.name }}" + register: new_workers + until: new_workers | json_query('resources[0].status.readyReplicas') | int == new_num_workers | int + retries: 40 + delay: 30 diff --git a/collection/stages/roles/install/tasks/main.yml b/collection/stages/roles/install/tasks/main.yml index a776e7d8..459cace7 100644 --- a/collection/stages/roles/install/tasks/main.yml +++ b/collection/stages/roles/install/tasks/main.yml @@ -22,6 +22,12 @@ state: directory mode: u=rwx,g=rw,o=r +- name: Discover Nova and Cinder AZs for install-config generation + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: discover_az.yml + when: az_enable | default(false) | bool + - name: Prepare for UPI installation ansible.builtin.include_role: name: tools_upi diff --git a/collection/stages/roles/install/templates/install-config-ipi.yaml.j2 b/collection/stages/roles/install/templates/install-config-ipi.yaml.j2 index fa62c376..eb1e97c4 100644 --- a/collection/stages/roles/install/templates/install-config-ipi.yaml.j2 +++ b/collection/stages/roles/install/templates/install-config-ipi.yaml.j2 @@ -7,7 +7,21 @@ controlPlane: openstack: type: "{{ installcfg_master_flavor }}" serverGroupPolicy: "{{ installcfg_master_servergrouppolicy }}" - {%- if installcfg_ctrl_rootvolume != {} +%} + {%- if az_enable | default(false) +%} + zones: {{ nova_az_for_masters | default([]) | to_json }} + {%- endif +%} + {%- if openshift_root_volumes | default(false) and az_enable | default(false) +%} + rootVolume: + size: {{ ocp_deployment_topology.controlplane_rootvolume.size | default(30) }} + # One volume type per zone, matched by AZ index so the installer can + # pair types[i] with zones[i]. Type for "AZ-" is "" + # (e.g. AZ-0 -> fastpool-0), matching the per-AZ Cinder backends. + types: +{%- for zone in cinder_az_for_masters | default([]) +%} + - {{ az_volume_type_prefix | default('fastpool-') }}{{ zone | regex_replace('^AZ-', '') }} +{%- endfor +%} + zones: {{ cinder_az_for_masters | default([]) | to_json }} + {%- elif installcfg_ctrl_rootvolume != {} +%} rootVolume: {{ installcfg_ctrl_rootvolume }} {%- endif +%} replicas: {{ installcfg_master_replicas }} @@ -17,6 +31,21 @@ compute: openstack: type: "{{ installcfg_worker_flavor }}" serverGroupPolicy: "{{ installcfg_worker_servergrouppolicy }}" + {%- if az_enable | default(false) +%} + zones: {{ nova_az_for_workers | default([]) | to_json }} + {%- endif +%} + {%- if openshift_root_volumes | default(false) and az_enable | default(false) +%} + rootVolume: + size: {{ ocp_deployment_topology.controlplane_rootvolume.size | default(30) }} + # One volume type per zone, matched by AZ index so the installer can + # pair types[i] with zones[i]. Type for "AZ-" is "" + # (e.g. AZ-0 -> fastpool-0), matching the per-AZ Cinder backends. + types: +{%- for zone in cinder_az_for_workers | default([]) +%} + - {{ az_volume_type_prefix | default('fastpool-') }}{{ zone | regex_replace('^AZ-', '') }} +{%- endfor +%} + zones: {{ cinder_az_for_workers | default([]) | to_json }} + {%- endif +%} {%- if ocp_deployment_topology.secondary_ip_protocol == 'ipv6' +%} additionalNetworkIDs: {{ registered_resources.ipv6_secondary_network_ids | to_json }} {%- endif +%} @@ -48,6 +77,9 @@ platform: {{ key }}: {{ value }} {%- endfor +%} {%- endif +%} + {%- if openshift_ipi_preload_rhcos_image | default(false) | bool +%} + clusterOSImage: rhcos + {%- endif +%} {%- if installcfg_machines_subnet +%} machinesSubnet: {{ installcfg_machines_subnet }} {%- endif +%} diff --git a/collection/stages/roles/prepare/defaults/main.yml b/collection/stages/roles/prepare/defaults/main.yml index 0e31dc8d..60d77832 100644 --- a/collection/stages/roles/prepare/defaults/main.yml +++ b/collection/stages/roles/prepare/defaults/main.yml @@ -38,3 +38,17 @@ etc_hosts_entries: - {regex: 'grafana-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} grafana-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'} - {regex: 'prometheus-k8s-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} prometheus-k8s-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'} - {regex: 'thanos-querier-openshift-monitoring.apps\..*\.{{ ocp_base_domain }}', row: '{{ apps_ip }} thanos-querier-openshift-monitoring.apps.{{ ocp_cluster_name }}.{{ ocp_base_domain }}'} + +az_enable: false +openshift_root_volumes: false +az_remove_on_cleanup: false +openshift_ipi_preload_rhcos_image: false +az_volume_type_prefix: "fastpool-" +az_backend_prefix: "backend_AZ" +az_volume_type_count: 3 +az_root_volume_types: + - fastpool-0 + - fastpool-1 + - fastpool-2 +az_configure_ceph_crush: true +az_compute_ssh_user: cloud-admin diff --git a/collection/stages/roles/prepare/tasks/configure_az.yml b/collection/stages/roles/prepare/tasks/configure_az.yml new file mode 100644 index 00000000..a1e079d3 --- /dev/null +++ b/collection/stages/roles/prepare/tasks/configure_az.yml @@ -0,0 +1,82 @@ +--- +- name: Discover nova compute service hosts + ansible.builtin.command: + cmd: openstack compute service list --service nova-compute -c Host -f value + environment: + OS_CLOUD: "{{ admin_cloud }}" + register: az_compute_services + changed_when: false + +- name: Set sorted compute host list for AZ configuration + ansible.builtin.set_fact: + az_compute_hosts: >- + {{ + az_compute_services.stdout_lines + | map('trim') + | reject('equalto', '') + | sort + | list + }} + +- name: Fail when fewer than three compute hosts are available for AZ + ansible.builtin.assert: + that: + - az_compute_hosts | length >= az_volume_type_count | int + fail_msg: >- + Expected at least {{ az_volume_type_count }} compute hosts for AZ setup, + found {{ az_compute_hosts | length }}: {{ az_compute_hosts }} + +# Derive the AZ index from the (sorted) position of the host in the list, not +# from the hostname. Compute service Host values may be FQDNs that do not end +# in a digit, in which case parsing a numeric suffix would produce AZ names +# that do not match the "AZ-" Cinder backends configured in the deployment. +- name: Create one host aggregate per compute host and assign its AZ + openstack.cloud.host_aggregate: + cloud: "{{ admin_cloud }}" + state: present + name: "HA-{{ az_idx }}" + availability_zone: "AZ-{{ az_idx }}" + hosts: + - "{{ item }}" + loop: "{{ az_compute_hosts }}" + loop_control: + index_var: az_idx + label: "{{ item }} -> AZ-{{ az_idx }}" + +- name: Show nova host AZ mapping + ansible.builtin.command: + cmd: openstack host list -c Zone -c "Host Name" -f table + environment: + OS_CLOUD: "{{ admin_cloud }}" + register: az_host_list + changed_when: false + +- name: Print nova host AZ mapping + ansible.builtin.debug: + var: az_host_list.stdout_lines + +- name: Delete existing AZ volume types if present + ansible.builtin.command: + cmd: openstack volume type delete {{ az_volume_type_prefix }}{{ item }} + environment: + OS_CLOUD: "{{ admin_cloud }}" + loop: "{{ range(0, az_volume_type_count | int) | list }}" + register: az_delete_volume_types + failed_when: false + changed_when: az_delete_volume_types.rc == 0 + +- name: Create AZ volume types + ansible.builtin.command: + cmd: >- + openstack volume type create + --property volume_backend_name={{ az_backend_prefix }}{{ item }} + {{ az_volume_type_prefix }}{{ item }} + environment: + OS_CLOUD: "{{ admin_cloud }}" + loop: "{{ range(0, az_volume_type_count | int) | list }}" + when: openshift_release is version('4.13', '>=') + changed_when: true + +- name: Configure Ceph crush rules for per-AZ pools + ansible.builtin.include_tasks: configure_az_ceph_crush.yml + when: az_configure_ceph_crush | bool diff --git a/collection/stages/roles/prepare/tasks/configure_az_ceph_crush.yml b/collection/stages/roles/prepare/tasks/configure_az_ceph_crush.yml new file mode 100644 index 00000000..3bf7f2c5 --- /dev/null +++ b/collection/stages/roles/prepare/tasks/configure_az_ceph_crush.yml @@ -0,0 +1,50 @@ +--- +- name: Resolve SSH targets for compute hosts + ansible.builtin.command: + cmd: openstack hypervisor show {{ item }} -c host_ip -f value + environment: + OS_CLOUD: "{{ admin_cloud }}" + loop: "{{ az_compute_hosts }}" + register: az_hypervisor_ips + changed_when: false + +- name: Build compute SSH target list + ansible.builtin.set_fact: + az_compute_ssh_hosts: "{{ az_hypervisor_ips.results | map(attribute='stdout') | map('trim') | list }}" + +# Pin each per-AZ pool (fastpool-) to the OSDs that physically live on the +# corresponding compute host , so a volume created in AZ- stays local to +# that host. The OSD ids are discovered from the cluster (via `ceph osd +# ls-tree `) rather than assumed to equal the AZ index: OSD ids are +# cluster-assigned and are not guaranteed to be contiguous or one-per-host. +# +# NOTE: a per-host device class can only satisfy an `osd` failure domain (there +# is a single host in the class), so the rule uses `osd`. If the pool `size` +# exceeds the number of OSDs on the host the pool will not reach active+clean; +# tune the pool size per cluster if needed. +- name: Pin each fastpool to its host-local OSDs via Ceph crush rules + ansible.builtin.shell: | + set -o pipefail + SSH="ssh -i {{ openstack_key_file }} -o StrictHostKeyChecking=no {{ az_compute_ssh_user }}@{{ az_compute_ssh_hosts[az_idx] }}" + CEPH_HOST=$(${SSH} "hostname -s") + OSDS=$(${SSH} "sudo cephadm shell -- ceph osd ls-tree ${CEPH_HOST}" | tr -d '\r') + if [ -z "${OSDS}" ]; then + echo "No OSDs found on crush host ${CEPH_HOST} (compute host {{ item }})" >&2 + exit 1 + fi + for osd in ${OSDS}; do + ${SSH} "sudo cephadm shell -- ceph osd crush rm-device-class ${osd} 2>/dev/null || true" + ${SSH} "sudo cephadm shell -- ceph osd crush set-device-class pool{{ az_idx }}_class ${osd}" + done + ${SSH} "sudo cephadm shell -- ceph osd crush rule create-replicated pool{{ az_idx }}_rule default osd pool{{ az_idx }}_class 2>/dev/null || true" + ${SSH} "sudo cephadm shell -- ceph osd pool set {{ az_volume_type_prefix }}{{ az_idx }} crush_rule pool{{ az_idx }}_rule" + args: + executable: /bin/bash + loop: "{{ az_compute_hosts }}" + loop_control: + index_var: az_idx + label: "{{ item }} -> {{ az_volume_type_prefix }}{{ az_idx }}" + when: + - az_idx < az_compute_ssh_hosts | length + - az_compute_ssh_hosts[az_idx] | length > 0 + changed_when: true diff --git a/collection/stages/roles/prepare/tasks/main.yml b/collection/stages/roles/prepare/tasks/main.yml index 745c99f2..9c75fd30 100644 --- a/collection/stages/roles/prepare/tasks/main.yml +++ b/collection/stages/roles/prepare/tasks/main.yml @@ -29,6 +29,10 @@ - name: Create new Project ansible.builtin.include_tasks: project.yml +- name: Configure Nova and Cinder availability zones + ansible.builtin.include_tasks: configure_az.yml + when: az_enable | default(false) | bool + - name: Update clouds.yml file with new Project ansible.builtin.include_tasks: clouds.yml diff --git a/collection/stages/roles/prepare/tasks/project.yml b/collection/stages/roles/prepare/tasks/project.yml index 2f5b7de2..018c6225 100644 --- a/collection/stages/roles/prepare/tasks/project.yml +++ b/collection/stages/roles/prepare/tasks/project.yml @@ -47,7 +47,7 @@ port: "{{ project.os_quota.port | default(omit) }}" volumes: "{{ project.os_quota.volumes | default(omit) }}" snapshots: "{{ project.os_quota.snapshots | default(omit) }}" - gigabytes: "{{ project.os_quota.gigabytes | default(omit) }}" + gigabytes: "{{ project.os_quota.gigabytes | default(az_enable | default(false) | ternary(2000, omit)) }}" verify: "{{ admin_verify_cacert }}" - name: Create flavors diff --git a/collection/stages/roles/verification/tasks/check_azs.yml b/collection/stages/roles/verification/tasks/check_azs.yml new file mode 100644 index 00000000..4130b4c2 --- /dev/null +++ b/collection/stages/roles/verification/tasks/check_azs.yml @@ -0,0 +1,143 @@ +--- +- name: Reset AZ verification facts + ansible.builtin.set_fact: + current_nova_az_for_masters: [] + current_nova_az_for_workers: [] + current_cinder_az_for_masters: [] + current_cinder_az_for_workers: [] + cinder_volumes_for_masters: [] + cinder_volumes_for_workers: [] + +- name: Gather OpenStack server details for the cluster + openstack.cloud.server_info: + cloud: "{{ user_cloud }}" + register: az_openstack_servers + changed_when: false + +- name: Store sorted master and worker server info + ansible.builtin.set_fact: + az_masters: "{{ az_openstack_servers.servers | selectattr('name', 'search', 'master') | sort(attribute='name') | list }}" + az_workers: "{{ az_openstack_servers.servers | selectattr('name', 'search', 'worker') | sort(attribute='name') | list }}" + +- name: Discover expected AZ distribution for install + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: discover_az.yml + vars: + ocp_deployment_topology: >- + {{ + ocp_deployment_topology | combine( + { + 'replicas': { + 'master': az_masters | length, + 'worker': az_workers | length + } + }, + recursive=True + ) + }} + +- name: Set expected nova AZ assignments + ansible.builtin.set_fact: + expected_nova_az_for_masters: "{{ nova_az_for_masters }}" + expected_nova_az_for_workers: "{{ nova_az_for_workers }}" + +- name: Set expected cinder AZ assignments + ansible.builtin.set_fact: + expected_cinder_az_for_masters: "{{ cinder_az_for_masters }}" + expected_cinder_az_for_workers: "{{ cinder_az_for_workers }}" + when: openshift_root_volumes | default(false) | bool + +- name: Collect current nova AZs for masters + ansible.builtin.set_fact: + current_nova_az_for_masters: "{{ current_nova_az_for_masters + [item.location.zone] }}" + loop: "{{ az_masters }}" + +- name: Collect current nova AZs for workers + ansible.builtin.set_fact: + current_nova_az_for_workers: "{{ current_nova_az_for_workers + [item.location.zone] }}" + loop: "{{ az_workers }}" + +- name: Collect Cinder volume AZ data for masters + when: openshift_root_volumes | default(false) | bool + block: + - name: Collect master root volume IDs + ansible.builtin.set_fact: + cinder_volumes_for_masters: "{{ cinder_volumes_for_masters + [item.volumes[0].id] }}" + loop: "{{ az_masters }}" + when: item.volumes | length > 0 + + - name: Query Cinder AZ for master root volumes + ansible.builtin.command: + cmd: openstack volume show {{ item }} -c availability_zone -f value + environment: + OS_CLOUD: "{{ user_cloud }}" + loop: "{{ cinder_volumes_for_masters }}" + register: az_master_volume_azs + changed_when: false + + - name: Store master Cinder AZ list + ansible.builtin.set_fact: + current_cinder_az_for_masters: "{{ current_cinder_az_for_masters + [item.stdout | trim] }}" + loop: "{{ az_master_volume_azs.results }}" + +- name: Collect Cinder volume AZ data for workers + when: openshift_root_volumes | default(false) | bool + block: + - name: Collect worker root volume IDs + ansible.builtin.set_fact: + cinder_volumes_for_workers: "{{ cinder_volumes_for_workers + [item.volumes[0].id] }}" + loop: "{{ az_workers }}" + when: item.volumes | length > 0 + + - name: Query Cinder AZ for worker root volumes + ansible.builtin.command: + cmd: openstack volume show {{ item }} -c availability_zone -f value + environment: + OS_CLOUD: "{{ user_cloud }}" + loop: "{{ cinder_volumes_for_workers }}" + register: az_worker_volume_azs + changed_when: false + + - name: Store worker Cinder AZ list + ansible.builtin.set_fact: + current_cinder_az_for_workers: "{{ current_cinder_az_for_workers + [item.stdout | trim] }}" + loop: "{{ az_worker_volume_azs.results }}" + +- name: Verify masters landed in expected nova AZs + ansible.builtin.assert: + that: + - current_nova_az_for_masters | sort == expected_nova_az_for_masters | sort + fail_msg: | + Unexpected nova AZs on masters: + obtained: {{ current_nova_az_for_masters | sort }} + expected: {{ expected_nova_az_for_masters | sort }} + +- name: Verify workers landed in expected nova AZs + ansible.builtin.assert: + that: + - current_nova_az_for_workers | sort == expected_nova_az_for_workers | sort + fail_msg: | + Unexpected nova AZs on workers: + obtained: {{ current_nova_az_for_workers | sort }} + expected: {{ expected_nova_az_for_workers | sort }} + +- name: Verify masters landed in expected Cinder AZs + ansible.builtin.assert: + that: + - current_cinder_az_for_masters | sort == expected_cinder_az_for_masters | sort + fail_msg: | + Unexpected cinder AZs on masters: + obtained: {{ current_cinder_az_for_masters | sort }} + expected: {{ expected_cinder_az_for_masters | sort }} + when: openshift_root_volumes | default(false) | bool + +- name: Verify workers landed in expected Cinder AZs + ansible.builtin.assert: + that: + - current_cinder_az_for_workers | sort == expected_cinder_az_for_workers | sort + fail_msg: | + Unexpected cinder AZs on workers: + obtained: {{ current_cinder_az_for_workers | sort }} + expected: {{ expected_cinder_az_for_workers | sort }} + when: openshift_root_volumes | default(false) | bool diff --git a/collection/stages/roles/verification/tasks/main.yml b/collection/stages/roles/verification/tasks/main.yml index 394f8120..052e61f9 100644 --- a/collection/stages/roles/verification/tasks/main.yml +++ b/collection/stages/roles/verification/tasks/main.yml @@ -101,6 +101,12 @@ - name: Create a demo app and check connectivity ansible.builtin.include_tasks: check_demo_app.yml + - name: Run Availability Zone checks + ansible.builtin.include_tasks: check_azs.yml + when: + - az_enable | default(false) | bool + - not _skip_health + - name: Create a LB type svc and check connectivity vars: # Using internal-lb annotation when cluster is deployed on restricted or provider network (FIPless): diff --git a/jobs_definitions/availability_zones_4.22_nightly.yaml b/jobs_definitions/availability_zones_4.22_nightly.yaml new file mode 100644 index 00000000..ee3b4dd4 --- /dev/null +++ b/jobs_definitions/availability_zones_4.22_nightly.yaml @@ -0,0 +1,76 @@ +--- +# +# * AVAILABILITY ZONES - OCP 4.22 NIGHTLY * +# +# - Port of DFG-osasinfra-shiftstack_periodic-multijob-availabilityzones +# - OSP with Nova + Cinder AZs; OCP IPI with rootVolumes per AZ +# - Runs on RHOSO 18.0 OSASINFRA Validated Architecture (serval70) +# +# Legacy ref: +# rhos-infrared/.../periodic-availabilityzones-env-vars-no-replace-master.groovy +# + +openshift_release: "4.22" +openshift_build_name: "" +installation_type: ipi + +az_enable: true +openshift_root_volumes: true +az_remove_on_cleanup: true +openshift_ipi_preload_rhcos_image: true + +stages: + - prepare + - install + - post + - verification + - day2ops + - openstack_test + - conformance_test + - cinder_csi_tests + - cpms_test + +conformance_test_type: serial + +day2ops_procedures: + - moving-etcd-to-ephemeral + - cpms_replace_attrs + - scaleup_worker + - cinder_registry + +ocp_deployment_topology: + network_type: OVNKubernetes + primary_ip_protocol: ipv4 + secondary_ip_protocol: "" + ipv4: + ip_version: 4 + tenant_subnet: + cidr: 10.196.0.0/16 + cluster_network: + cidr: 10.128.0.0/14 + host_prefix: 23 + service_network: + - 172.30.0.0/16 + flavors: + master: + name: "master" + ram: 16384 + vcpus: 4 + disk: 30 + ephemeral: 10 + worker: + name: "worker" + ram: 8192 + vcpus: 2 + disk: 30 + replicas: + master: 3 + worker: 2 + servergroups: + master: "soft-anti-affinity" + worker: "soft-anti-affinity" + controlplane_rootvolume: + # Root volume size for AZ rootVolumes; per-AZ volume types and zones are + # derived at install time from the discovered Cinder AZs, so no static + # types list is needed here. + size: 30