diff --git a/.ci-operator.yaml b/.ci-operator.yaml index 8f00d38a..69a16ee6 100644 --- a/.ci-operator.yaml +++ b/.ci-operator.yaml @@ -1,4 +1,4 @@ build_root_image: name: boilerplate namespace: openshift - tag: image-v8.4.1 + tag: image-v8.4.3 diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 004cb068..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "docker" - directory: "/build" - labels: - - "area/dependency" - - "ok-to-test" - schedule: - interval: "weekly" - ignore: - - dependency-name: "app-sre/boilerplate" - # don't upgrade boilerplate via these means - - dependency-name: "openshift4/ose-operator-registry" - # don't upgrade ose-operator-registry via these means diff --git a/.tekton/managed-node-metadata-operator-agentic-sdlc-check-pull-request.yaml b/.tekton/managed-node-metadata-operator-agentic-sdlc-check-pull-request.yaml index c1efbd82..f9ac779f 100644 --- a/.tekton/managed-node-metadata-operator-agentic-sdlc-check-pull-request.yaml +++ b/.tekton/managed-node-metadata-operator-agentic-sdlc-check-pull-request.yaml @@ -43,7 +43,7 @@ spec: - name: url value: https://github.com/openshift/boilerplate - name: revision - value: 663c1d851e3bcb533a7be50936a0d55c732b7321 + value: b361d1a597f5a3a840ccea7efd40dc92f8cfa789 - name: pathInRepo value: pipelines/agentic-sdlc-check/pipeline.yaml status: {} diff --git a/OWNERS b/OWNERS index 05ed974e..84668a60 100644 --- a/OWNERS +++ b/OWNERS @@ -3,6 +3,7 @@ approvers: - bergmannf - tkong-redhat - rbhilare +- srep-functional-team-hulk - rosa-staff-engineers reviewers: - iamkirkbater diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index d8d63eba..19dc0e2d 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -38,7 +38,6 @@ aliases: - Makdaam - Nikokolas3270 - RaphaelBut - - MateSaary - rolandmkunkel - petrkotas - zmird-r @@ -56,7 +55,6 @@ aliases: - feichashao - samanthajayasinghe - xiaoyu74 - - Tessg22 - smarthall rosa-staff-engineers: - Ajpantuso @@ -65,6 +63,7 @@ aliases: - bpresnel-rh - clcollins - dustman9000 + - geowa4 - joshbranham - psav - rafael-azevedo @@ -75,7 +74,6 @@ aliases: - typeid rosa-managers: - afreiberger - - c-e-brumm - drewandersonnz - evalis1 - geowa4 @@ -83,10 +81,7 @@ aliases: - jnewton75 - krishvoor - kseiter-rh - - OliviaHY - - syncrou - tdrozdowski - - tkiss28 - vkumar51 hp-architects: - deads2k diff --git a/boilerplate/_data/backing-image-tag b/boilerplate/_data/backing-image-tag index f7a28f51..284dd19e 100644 --- a/boilerplate/_data/backing-image-tag +++ b/boilerplate/_data/backing-image-tag @@ -1 +1 @@ -image-v8.4.1 +image-v8.4.3 diff --git a/boilerplate/_data/last-boilerplate-commit b/boilerplate/_data/last-boilerplate-commit index 3ae4abbc..1d4e057b 100644 --- a/boilerplate/_data/last-boilerplate-commit +++ b/boilerplate/_data/last-boilerplate-commit @@ -1 +1 @@ -663c1d851e3bcb533a7be50936a0d55c732b7321 +b361d1a597f5a3a840ccea7efd40dc92f8cfa789 diff --git a/boilerplate/openshift/golang-osd-e2e/README.md b/boilerplate/openshift/golang-osd-e2e/README.md index cb747a68..cc7ba3d1 100644 --- a/boilerplate/openshift/golang-osd-e2e/README.md +++ b/boilerplate/openshift/golang-osd-e2e/README.md @@ -31,8 +31,48 @@ following: |------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `e2e-binary-build` | Compiles ginkgo tests under test/e2e and creates the ginkgo binary. | | `e2e-image-build-push` | Builds e2e image and pushes to operator's quay repo. Image name is defaulted to -test-harness. Quay repository must be created beforehand. | +| `e2e-local` | Builds the e2e binary and runs it against a cluster via KUBECONFIG or backplane. Supports focused tests via GINKGO_FOCUS. | #### E2E Local Testing -Please follow [this README](https://github.com/openshift/ops-sop/blob/master/v4/howto/osde2e/operator-test-harnesses.md#using-ginkgo) to run your e2e tests locally +Run e2e tests locally against a managed cluster without waiting for the full Prow CI pipeline. +**Prerequisites:** +- `ocm` CLI logged into the appropriate environment (`ocm login --use-auth-code --url staging`) +- Access to a managed cluster (via KUBECONFIG or backplane) +- Go toolchain installed + +**Option 1: Using backplane (recommended)** + +```bash +# Run all tests against a cluster by ID +make e2e-local CLUSTER_ID=2rmlgv5dbdp2285n85o7h3aaa6pafkpq + +# Run focused tests +make e2e-local CLUSTER_ID=2rmlgv5dbdp2285n85o7h3aaa6pafkpq GINKGO_FOCUS="is installed" + +# Run tests with a label filter +make e2e-local CLUSTER_ID=2rmlgv5dbdp2285n85o7h3aaa6pafkpq GINKGO_LABEL_FILTER="!slow" +``` + +**Option 2: Using an existing KUBECONFIG** + +```bash +# Set KUBECONFIG to your cluster's kubeconfig +export KUBECONFIG=/path/to/kubeconfig + +# Run all tests +make e2e-local + +# Run a single test +make e2e-local GINKGO_FOCUS="reconciles required resources" +``` + +**Output:** +- Test results print to stdout with verbose Ginkgo output +- JUnit XML report saved to `e2e-local-junit.xml` + +**Tips:** +- Use lease clusters for testing (check `#rosa-prow-info` for available clusters) +- The operator must be deployed on the target cluster (via PKO or OLM) +- If tests fail with "not found" errors, verify the operator is running: `oc get deployment -n ` diff --git a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml index ec701e6d..d1a925a4 100644 --- a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml +++ b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml @@ -8,11 +8,20 @@ parameters: required: true description: Prow periodic job name to trigger via Gangway - name: POLL_INTERVAL - value: "60" + value: "120" description: Seconds between status polls - name: TIMEOUT value: "7200" - description: Maximum seconds to wait for job completion + description: Maximum seconds to wait per attempt for job completion + - name: MAX_RETRIES + value: "5" + description: Number of times to retry the Prow job on failure before reporting failure + - name: ACTIVE_DEADLINE + value: "54000" + description: Kubernetes Job deadline in seconds (must exceed all attempts plus backoff delays) + - name: INITIAL_DELAY + value: "0" + description: Seconds to sleep before the first Gangway call; stagger concurrent jobs to avoid shared rate limit saturation - name: JOB_ENVS value: "" description: Comma-separated KEY=VALUE pairs passed to the Prow job @@ -29,7 +38,7 @@ objects: name: gangway-bridge-${IMAGE_TAG}-${JOBID} spec: backoffLimit: 0 - activeDeadlineSeconds: ${{TIMEOUT}} + activeDeadlineSeconds: ${{ACTIVE_DEADLINE}} template: spec: automountServiceAccountToken: false @@ -46,6 +55,27 @@ objects: [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } + [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + [[ "${INITIAL_DELAY}" =~ ^[0-9]+$ ]] || { log "ERROR: INITIAL_DELAY must be a non-negative integer"; exit 1; } + + if [[ "${INITIAL_DELAY}" -gt 0 ]]; then + log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." + sleep "${INITIAL_DELAY}" + fi + + # Backoff sum: base 30s doubling each retry, capped at 900s, plus 15s max jitter + MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) + # Each attempt may overshoot TIMEOUT by up to max(POLL_INTERVAL, 300s max backoff) + + # status-request max-time (30s) on the last poll cycle + POLL_OVERSHOOT=$(( (POLL_INTERVAL > 300 ? POLL_INTERVAL : 300) + 30 )) + # Trigger POST max-time (60s) + worst-case Retry-After (600s) per attempt + TRIGGER_OVERHEAD=$(( 60 + 600 )) + # INITIAL_DELAY is a one-time cost at job startup, not per attempt + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM + INITIAL_DELAY )) + if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then + log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" + exit 1 + fi BODY='{"job_execution_type":"1"}' if [[ -n "${JOB_ENVS:-}" ]]; then @@ -53,21 +83,99 @@ objects: BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') fi - RESP=$(curl -sfSL --retry 3 --retry-delay 10 -X POST -H "Authorization: Bearer ${GANGWAY_TOKEN}" -H "Content-Type: application/json" -d "${BODY}" "${GW}/${JOB_NAME}") - ID=$(echo "$RESP" | jq -re .id) - PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" - log "Triggered ${JOB_NAME} -> ${ID}" - log "Prow logs: ${PROW_URL}" + RATE_LIMITED_WAITED=0 + trigger_and_poll() { + local resp_file="/dev/shm/gw_resp.$$" header_file="/dev/shm/gw_hdr.$$" + trap 'rm -f "$resp_file" "$header_file"' RETURN + HTTP_CODE=$(curl -sSL --max-time 60 -X POST \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${BODY}" \ + -o "$resp_file" -D "$header_file" \ + -w '%{http_code}' "${GW}/${JOB_NAME}" 2>/dev/null) || HTTP_CODE=000 + + # Handle 429 — parse Retry-After header (capped at 600s) + if [[ "$HTTP_CODE" == "429" ]]; then + local retry_after + retry_after=$(grep -i '^retry-after:' "$header_file" | awk '{print $2}' | tr -d '\r') + if [[ "$retry_after" =~ ^[0-9]+$ ]] && [[ "$retry_after" -gt 0 ]] && [[ "$retry_after" -le 600 ]]; then + log "Rate limited (429) — sleeping ${retry_after}s (Retry-After)" + sleep "$retry_after" + RATE_LIMITED_WAITED=1 + else + log "Rate limited (429) — no valid Retry-After header" + fi + return 1 # falls through to outer retry with backoff + fi + + # Fail on non-2xx + if [[ "$HTTP_CODE" -lt 200 || "$HTTP_CODE" -ge 300 ]]; then + log "Failed to trigger ${JOB_NAME} (HTTP ${HTTP_CODE})" + return 1 + fi + + RESP=$(cat "$resp_file") + if ! ID=$(echo "$RESP" | jq -re .id); then + log "Gangway did not return a valid execution ID" + return 1 + fi + PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" + log "Triggered ${JOB_NAME} -> ${ID}" + log "Prow logs: ${PROW_URL}" + + END=$((SECONDS + ${TIMEOUT})) + local poll_backoff="${POLL_INTERVAL}" + while [[ $SECONDS -lt $END ]]; do + sleep "$poll_backoff" + local poll_file="/dev/shm/gw_poll.$$" + local poll_code + poll_code=$(curl -sSL --max-time 30 \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -o "$poll_file" -w '%{http_code}' \ + "${GW}/${ID}" 2>/dev/null) || poll_code=000 + if [[ "$poll_code" == "429" ]]; then + rm -f "$poll_file" + poll_backoff=$(( poll_backoff * 2 )) + [[ $poll_backoff -gt 300 ]] && poll_backoff=300 + log "Rate limited polling status (429) — backing off ${poll_backoff}s" + continue + fi + poll_backoff="${POLL_INTERVAL}" + S=$(jq -r .job_status "$poll_file" 2>/dev/null) || S=UNKNOWN + rm -f "$poll_file" + log "${S} ($((SECONDS))s)" + case $S in + SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; + FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; return 1;; + esac + done + log "Prow logs: ${PROW_URL}" + log "Timeout"; return 1 + } - END=$((SECONDS + ${TIMEOUT})) - while [[ $SECONDS -lt $END ]]; do - sleep "${POLL_INTERVAL}" - S=$(curl -sfSL -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN - log "${S} ($((SECONDS))s)" - case $S in SUCCESS) log "Prow logs: ${PROW_URL}"; exit 0;; FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; exit 1;; esac + ATTEMPT=0 + while true; do + ATTEMPT=$((ATTEMPT + 1)) + log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" + if trigger_and_poll; then + exit 0 + fi + if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then + log "All attempts exhausted" + exit 1 + fi + if [[ $RATE_LIMITED_WAITED -eq 1 ]]; then + log "Skipping backoff (already waited for Retry-After)" + RATE_LIMITED_WAITED=0 + else + BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) + [[ $BACKOFF -gt 900 ]] && BACKOFF=900 + JITTER=$(( RANDOM % 16 )) + DELAY=$(( BACKOFF + JITTER )) + log "Retrying in ${DELAY}s (backoff=${BACKOFF}s, jitter=${JITTER}s)..." + sleep "$DELAY" + fi done - log "Prow logs: ${PROW_URL}" - log "Timeout"; exit 1 env: - name: JOB_NAME value: ${JOB_NAME} @@ -82,6 +190,12 @@ objects: value: ${TIMEOUT} - name: JOB_ENVS value: ${JOB_ENVS} + - name: MAX_RETRIES + value: ${MAX_RETRIES} + - name: INITIAL_DELAY + value: ${INITIAL_DELAY} + - name: ACTIVE_DEADLINE + value: ${ACTIVE_DEADLINE} resources: requests: cpu: "50m" diff --git a/boilerplate/openshift/golang-osd-e2e/standard.mk b/boilerplate/openshift/golang-osd-e2e/standard.mk index 7d051307..22f94aae 100644 --- a/boilerplate/openshift/golang-osd-e2e/standard.mk +++ b/boilerplate/openshift/golang-osd-e2e/standard.mk @@ -64,6 +64,26 @@ e2e-binary-build: go mod tidy go test ./test/e2e -v -c --tags=osde2e -o e2e.test +# Run e2e tests locally against a cluster accessible via KUBECONFIG. +# Usage: +# make e2e-local # run all tests +# make e2e-local GINKGO_FOCUS="test name" # run matching tests +# make e2e-local CLUSTER_ID= # use backplane for cluster access +# +# Requires: KUBECONFIG set, or CLUSTER_ID + ocm login for backplane access. +.PHONY: e2e-local +e2e-local: e2e-binary-build + @if [ -n "$(CLUSTER_ID)" ] && [ -z "$(KUBECONFIG)" ]; then \ + echo "Logging into cluster $(CLUSTER_ID) via backplane..."; \ + ocm backplane login $(CLUSTER_ID); \ + fi + @echo "Running e2e tests against $${KUBECONFIG:-backplane cluster}..." + DISABLE_JUNIT_REPORT=true ./e2e.test \ + --ginkgo.v \ + --ginkgo.junit-report=e2e-local-junit.xml \ + $(if $(GINKGO_FOCUS),--ginkgo.focus="$(GINKGO_FOCUS)") \ + $(if $(GINKGO_LABEL_FILTER),--ginkgo.label-filter="$(GINKGO_LABEL_FILTER)") + # push e2e image tagged as latest and as repo commit hash .PHONY: e2e-image-build-push e2e-image-build-push: container-engine-login diff --git a/boilerplate/openshift/golang-osd-operator/OWNERS_ALIASES b/boilerplate/openshift/golang-osd-operator/OWNERS_ALIASES index d8d63eba..19dc0e2d 100644 --- a/boilerplate/openshift/golang-osd-operator/OWNERS_ALIASES +++ b/boilerplate/openshift/golang-osd-operator/OWNERS_ALIASES @@ -38,7 +38,6 @@ aliases: - Makdaam - Nikokolas3270 - RaphaelBut - - MateSaary - rolandmkunkel - petrkotas - zmird-r @@ -56,7 +55,6 @@ aliases: - feichashao - samanthajayasinghe - xiaoyu74 - - Tessg22 - smarthall rosa-staff-engineers: - Ajpantuso @@ -65,6 +63,7 @@ aliases: - bpresnel-rh - clcollins - dustman9000 + - geowa4 - joshbranham - psav - rafael-azevedo @@ -75,7 +74,6 @@ aliases: - typeid rosa-managers: - afreiberger - - c-e-brumm - drewandersonnz - evalis1 - geowa4 @@ -83,10 +81,7 @@ aliases: - jnewton75 - krishvoor - kseiter-rh - - OliviaHY - - syncrou - tdrozdowski - - tkiss28 - vkumar51 hp-architects: - deads2k diff --git a/boilerplate/openshift/golang-osd-operator/codecov.sh b/boilerplate/openshift/golang-osd-operator/codecov.sh index 8fc79bd0..95fb5a90 100755 --- a/boilerplate/openshift/golang-osd-operator/codecov.sh +++ b/boilerplate/openshift/golang-osd-operator/codecov.sh @@ -23,11 +23,11 @@ rm -f "${COVER_PROFILE}.tmp" # Configure the git refs and job link based on how the job was triggered via prow if [[ "${JOB_TYPE}" == "presubmit" ]]; then echo "detected PR code coverage job for #${PULL_NUMBER}" - REF_FLAGS="-P ${PULL_NUMBER} -C ${PULL_PULL_SHA}" + REF_FLAGS="--pr ${PULL_NUMBER} --commit-sha ${PULL_PULL_SHA}" JOB_LINK="${CI_SERVER_URL}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}" elif [[ "${JOB_TYPE}" == "postsubmit" ]]; then echo "detected branch code coverage job for ${PULL_BASE_REF}" - REF_FLAGS="-B ${PULL_BASE_REF} -C ${PULL_BASE_SHA}" + REF_FLAGS="--branch ${PULL_BASE_REF} --commit-sha ${PULL_BASE_SHA}" JOB_LINK="${CI_SERVER_URL}/logs/${JOB_NAME}/${BUILD_ID}" elif [[ "${JOB_TYPE}" == "local" ]]; then echo "coverage report available at ${COVER_PROFILE}" @@ -43,12 +43,17 @@ export CI_BUILD_ID="${JOB_NAME}" export CI_JOB_ID="${BUILD_ID}" if [[ "${JOB_TYPE}" != "local" ]]; then - if [[ -z "${ARTIFACT_DIR:-}" ]] || [[ ! -d "${ARTIFACT_DIR}" ]] || [[ ! -w "${ARTIFACT_DIR}" ]]; then - echo '${ARTIFACT_DIR} must be set for non-local jobs, and must point to a writable directory' >&2 - exit 1 - fi - curl -sS https://codecov.io/bash -o "${ARTIFACT_DIR}/codecov.sh" - bash <(cat "${ARTIFACT_DIR}/codecov.sh") -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} + CODECOV_VERSION="${CODECOV_VERSION:-v11.3.1}" + CODECOV_SHA256="${CODECOV_SHA256:-ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d}" + CODECOV_BIN="$(mktemp -d)/codecov" + + curl -sSfL "https://github.com/codecov/codecov-cli/releases/download/${CODECOV_VERSION}/codecovcli_linux" \ + -o "${CODECOV_BIN}" + echo "${CODECOV_SHA256} ${CODECOV_BIN}" | sha256sum -c - + chmod +x "${CODECOV_BIN}" + + "${CODECOV_BIN}" upload-process --fail-on-error --git-service github \ + --file "${COVER_PROFILE}" --slug "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} else - bash <(curl -s https://codecov.io/bash) -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} + echo "coverage report available at ${COVER_PROFILE} (no upload in local mode)" fi diff --git a/boilerplate/openshift/golang-osd-operator/dependabot.yml b/boilerplate/openshift/golang-osd-operator/dependabot.yml deleted file mode 100644 index eae3de41..00000000 --- a/boilerplate/openshift/golang-osd-operator/dependabot.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "docker" - directory: "/build" - labels: - - "area/dependency" - - "ok-to-test" - - "lgtm" - - "approved" - schedule: - interval: "weekly" - day: "monday" - time: "03:00" - timezone: "UTC" - ignore: - - dependency-name: "redhat-services-prod/openshift/boilerplate" - # don't upgrade boilerplate via these means - - dependency-name: "openshift4/ose-operator-registry" - # don't upgrade ose-operator-registry via these means diff --git a/boilerplate/openshift/golang-osd-operator/update b/boilerplate/openshift/golang-osd-operator/update index 7aa403a4..ff479267 100755 --- a/boilerplate/openshift/golang-osd-operator/update +++ b/boilerplate/openshift/golang-osd-operator/update @@ -76,30 +76,22 @@ if [[ ${#OWNERS_UPDATED[@]} -gt 0 ]]; then echo "=====================" fi -# Add dependabot configuration -mkdir -p $REPO_ROOT/.github +# Remove Dependabot config previously shipped by boilerplate. +# Konflux MintMaker owns dependency updates for operators; Dependabot duplicates that. TARGET_FILE="${REPO_ROOT}/.github/dependabot.yml" -BOILERPLATE_FILE="${HERE}/dependabot.yml" - if [[ -f "$TARGET_FILE" ]]; then if grep -q '# BEGIN boilerplate-managed' "$TARGET_FILE"; then - echo "Boilerplate-managed section already present in dependabot.yml, skipping append." - elif diff -q "$TARGET_FILE" "$BOILERPLATE_FILE" >/dev/null; then - echo "Wrapping existing dependabot.yml (which matches boilerplate) with boilerplate-managed markers..." - mv "$TARGET_FILE" "${TARGET_FILE}.bak" - { - echo "# BEGIN boilerplate-managed" - cat "${TARGET_FILE}.bak" - echo "# END boilerplate-managed" - } > "$TARGET_FILE" - rm -f "${TARGET_FILE}.bak" + echo "Removing boilerplate-managed .github/dependabot.yml (Konflux/MintMaker owns dependency updates)" + rm -f "$TARGET_FILE" + elif grep -qE 'package-ecosystem:[[:space:]]*"docker"' "$TARGET_FILE" \ + && grep -q 'directory: "/build"' "$TARGET_FILE" \ + && ! grep -qE 'package-ecosystem:[[:space:]]*("gomod"|gomod|"github-actions"|github-actions|"npm"|npm|"pip"|pip)' "$TARGET_FILE"; then + echo "Removing legacy boilerplate docker Dependabot config from .github/dependabot.yml" + rm -f "$TARGET_FILE" else - echo "[WARNING] dependabot.yml exists and differs from boilerplate template but has no boilerplate-managed markers." - echo "[WARNING] Please review manually to avoid config duplication." + echo "[WARNING] .github/dependabot.yml exists and may contain non-boilerplate config; not removing automatically." + echo "[WARNING] Review and remove Dependabot entries that duplicate Konflux/MintMaker if appropriate." fi -else - echo "Copying boilerplate-managed dependabot.yml" - cp "$BOILERPLATE_FILE" "$TARGET_FILE" fi # Add olm-registry Dockerfile diff --git a/build/Dockerfile b/build/Dockerfile index 4d12ea22..e40bb7c3 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/redhat-services-prod/openshift/boilerplate:image-v8.4.1 AS builder +FROM quay.io/redhat-services-prod/openshift/boilerplate:image-v8.4.3 AS builder WORKDIR /workspace # Copy the Go Modules manifests @@ -18,7 +18,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=mod -a -o # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1784705586 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1788166357 ENV USER_UID=1001 \ USER_NAME=managed-node-metadata-operator diff --git a/build/Dockerfile.olm-registry b/build/Dockerfile.olm-registry index 3de29c68..08bf2be5 100644 --- a/build/Dockerfile.olm-registry +++ b/build/Dockerfile.olm-registry @@ -4,7 +4,7 @@ COPY ${SAAS_OPERATOR_DIR} manifests RUN initializer --permissive # ubi-micro does not work for clusters with fips enabled unless we make OpenSSL available -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1784705586 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1788166357 COPY --from=builder /bin/registry-server /bin/registry-server COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe diff --git a/test/e2e/gangway-bridge-template.yml b/test/e2e/gangway-bridge-template.yml index ec701e6d..d1a925a4 100644 --- a/test/e2e/gangway-bridge-template.yml +++ b/test/e2e/gangway-bridge-template.yml @@ -8,11 +8,20 @@ parameters: required: true description: Prow periodic job name to trigger via Gangway - name: POLL_INTERVAL - value: "60" + value: "120" description: Seconds between status polls - name: TIMEOUT value: "7200" - description: Maximum seconds to wait for job completion + description: Maximum seconds to wait per attempt for job completion + - name: MAX_RETRIES + value: "5" + description: Number of times to retry the Prow job on failure before reporting failure + - name: ACTIVE_DEADLINE + value: "54000" + description: Kubernetes Job deadline in seconds (must exceed all attempts plus backoff delays) + - name: INITIAL_DELAY + value: "0" + description: Seconds to sleep before the first Gangway call; stagger concurrent jobs to avoid shared rate limit saturation - name: JOB_ENVS value: "" description: Comma-separated KEY=VALUE pairs passed to the Prow job @@ -29,7 +38,7 @@ objects: name: gangway-bridge-${IMAGE_TAG}-${JOBID} spec: backoffLimit: 0 - activeDeadlineSeconds: ${{TIMEOUT}} + activeDeadlineSeconds: ${{ACTIVE_DEADLINE}} template: spec: automountServiceAccountToken: false @@ -46,6 +55,27 @@ objects: [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } + [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + [[ "${INITIAL_DELAY}" =~ ^[0-9]+$ ]] || { log "ERROR: INITIAL_DELAY must be a non-negative integer"; exit 1; } + + if [[ "${INITIAL_DELAY}" -gt 0 ]]; then + log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." + sleep "${INITIAL_DELAY}" + fi + + # Backoff sum: base 30s doubling each retry, capped at 900s, plus 15s max jitter + MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) + # Each attempt may overshoot TIMEOUT by up to max(POLL_INTERVAL, 300s max backoff) + + # status-request max-time (30s) on the last poll cycle + POLL_OVERSHOOT=$(( (POLL_INTERVAL > 300 ? POLL_INTERVAL : 300) + 30 )) + # Trigger POST max-time (60s) + worst-case Retry-After (600s) per attempt + TRIGGER_OVERHEAD=$(( 60 + 600 )) + # INITIAL_DELAY is a one-time cost at job startup, not per attempt + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM + INITIAL_DELAY )) + if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then + log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" + exit 1 + fi BODY='{"job_execution_type":"1"}' if [[ -n "${JOB_ENVS:-}" ]]; then @@ -53,21 +83,99 @@ objects: BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') fi - RESP=$(curl -sfSL --retry 3 --retry-delay 10 -X POST -H "Authorization: Bearer ${GANGWAY_TOKEN}" -H "Content-Type: application/json" -d "${BODY}" "${GW}/${JOB_NAME}") - ID=$(echo "$RESP" | jq -re .id) - PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" - log "Triggered ${JOB_NAME} -> ${ID}" - log "Prow logs: ${PROW_URL}" + RATE_LIMITED_WAITED=0 + trigger_and_poll() { + local resp_file="/dev/shm/gw_resp.$$" header_file="/dev/shm/gw_hdr.$$" + trap 'rm -f "$resp_file" "$header_file"' RETURN + HTTP_CODE=$(curl -sSL --max-time 60 -X POST \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${BODY}" \ + -o "$resp_file" -D "$header_file" \ + -w '%{http_code}' "${GW}/${JOB_NAME}" 2>/dev/null) || HTTP_CODE=000 + + # Handle 429 — parse Retry-After header (capped at 600s) + if [[ "$HTTP_CODE" == "429" ]]; then + local retry_after + retry_after=$(grep -i '^retry-after:' "$header_file" | awk '{print $2}' | tr -d '\r') + if [[ "$retry_after" =~ ^[0-9]+$ ]] && [[ "$retry_after" -gt 0 ]] && [[ "$retry_after" -le 600 ]]; then + log "Rate limited (429) — sleeping ${retry_after}s (Retry-After)" + sleep "$retry_after" + RATE_LIMITED_WAITED=1 + else + log "Rate limited (429) — no valid Retry-After header" + fi + return 1 # falls through to outer retry with backoff + fi + + # Fail on non-2xx + if [[ "$HTTP_CODE" -lt 200 || "$HTTP_CODE" -ge 300 ]]; then + log "Failed to trigger ${JOB_NAME} (HTTP ${HTTP_CODE})" + return 1 + fi + + RESP=$(cat "$resp_file") + if ! ID=$(echo "$RESP" | jq -re .id); then + log "Gangway did not return a valid execution ID" + return 1 + fi + PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" + log "Triggered ${JOB_NAME} -> ${ID}" + log "Prow logs: ${PROW_URL}" + + END=$((SECONDS + ${TIMEOUT})) + local poll_backoff="${POLL_INTERVAL}" + while [[ $SECONDS -lt $END ]]; do + sleep "$poll_backoff" + local poll_file="/dev/shm/gw_poll.$$" + local poll_code + poll_code=$(curl -sSL --max-time 30 \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -o "$poll_file" -w '%{http_code}' \ + "${GW}/${ID}" 2>/dev/null) || poll_code=000 + if [[ "$poll_code" == "429" ]]; then + rm -f "$poll_file" + poll_backoff=$(( poll_backoff * 2 )) + [[ $poll_backoff -gt 300 ]] && poll_backoff=300 + log "Rate limited polling status (429) — backing off ${poll_backoff}s" + continue + fi + poll_backoff="${POLL_INTERVAL}" + S=$(jq -r .job_status "$poll_file" 2>/dev/null) || S=UNKNOWN + rm -f "$poll_file" + log "${S} ($((SECONDS))s)" + case $S in + SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; + FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; return 1;; + esac + done + log "Prow logs: ${PROW_URL}" + log "Timeout"; return 1 + } - END=$((SECONDS + ${TIMEOUT})) - while [[ $SECONDS -lt $END ]]; do - sleep "${POLL_INTERVAL}" - S=$(curl -sfSL -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN - log "${S} ($((SECONDS))s)" - case $S in SUCCESS) log "Prow logs: ${PROW_URL}"; exit 0;; FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; exit 1;; esac + ATTEMPT=0 + while true; do + ATTEMPT=$((ATTEMPT + 1)) + log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" + if trigger_and_poll; then + exit 0 + fi + if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then + log "All attempts exhausted" + exit 1 + fi + if [[ $RATE_LIMITED_WAITED -eq 1 ]]; then + log "Skipping backoff (already waited for Retry-After)" + RATE_LIMITED_WAITED=0 + else + BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) + [[ $BACKOFF -gt 900 ]] && BACKOFF=900 + JITTER=$(( RANDOM % 16 )) + DELAY=$(( BACKOFF + JITTER )) + log "Retrying in ${DELAY}s (backoff=${BACKOFF}s, jitter=${JITTER}s)..." + sleep "$DELAY" + fi done - log "Prow logs: ${PROW_URL}" - log "Timeout"; exit 1 env: - name: JOB_NAME value: ${JOB_NAME} @@ -82,6 +190,12 @@ objects: value: ${TIMEOUT} - name: JOB_ENVS value: ${JOB_ENVS} + - name: MAX_RETRIES + value: ${MAX_RETRIES} + - name: INITIAL_DELAY + value: ${INITIAL_DELAY} + - name: ACTIVE_DEADLINE + value: ${ACTIVE_DEADLINE} resources: requests: cpu: "50m"