From dea807e7b1f7301b7474ed9c5694e3432b58cf18 Mon Sep 17 00:00:00 2001 From: Parikshit Khedekar Date: Wed, 2 Sep 2026 13:18:40 +0530 Subject: [PATCH] OCPBUGS-115005: add polling timeout for Azure NIC updates The waitForCompletion function used context.TODO() with no timeout when polling Azure NIC update operations. This means each operation can block indefinitely while holding the per-node mutex, serializing all subsequent IP assignments for that node. On clusters with many EgressIPs per node (e.g. 250+), this causes cascading delays of 10+ minutes during upgrades when IPs are reassigned. Add a 2-minute polling timeout (defaultAzurePollingTimeout) to bound individual operations. This aligns with the AWS provider which uses a 1-minute timeout via PollUntilContextTimeout. The Azure timeout is set higher to account for Azure NIC updates involving more API roundtrips (7-8 vs 2 for AWS) and the SDK's default polling interval (~30s). Signed-off-by: Parikshit Khedekar Assisted-By: Claude Opus 4.6 --- pkg/cloudprovider/azure.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/azure.go b/pkg/cloudprovider/azure.go index fd2313a2f..0d73b9e1c 100644 --- a/pkg/cloudprovider/azure.go +++ b/pkg/cloudprovider/azure.go @@ -42,6 +42,10 @@ const ( defaultAzurePrivateIPCapacity = 256 // defaultAzureOperationTimeout is the timeout for all Azure operations defaultAzureOperationTimeout = 10 * time.Second + // defaultAzurePollingTimeout is the timeout for polling Azure NIC update + // operations to completion. Azure NIC updates involve multiple API + // roundtrips and can take significantly longer than the initial request. + defaultAzurePollingTimeout = 2 * time.Minute ) // Azure implements the API wrapper for talking @@ -392,11 +396,20 @@ func (a *Azure) createOrUpdate(networkInterface armnetwork.Interface) (*runtime. } func (a *Azure) waitForCompletion(poller *runtime.Poller[armnetwork.InterfacesClientCreateOrUpdateResponse]) error { - // No specified timeout for this operation, because a valid value doesn't - // seem possible to estimate. Note: Azure has some defaults defined here: + // Previously, no timeout was specified for this operation because a valid + // value didn't seem possible to estimate. Note: Azure has some defaults + // defined here: // https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime@v1.17.0#Poller.PollUntilDone - if _, err := poller.PollUntilDone(context.TODO(), nil); err != nil { - return err + // + // Cap the polling to defaultAzurePollingTimeout (2 min). Each NIC update + // involves 7-8 Azure API roundtrips; 2 min allows ~4 poll cycles at the + // SDK's default 30s interval, which is generous for a single IP add/remove. + // If the operation hasn't completed by then, return an error so the + // controller can retry and the per-node mutex is released. + ctx, cancel := context.WithTimeout(context.Background(), defaultAzurePollingTimeout) + defer cancel() + if _, err := poller.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("timed out or failed waiting for Azure NIC update to complete: %w", err) } return nil }