fix error setting password for app due to app not found

Author: mrWinston

pkg/util/cluster/aad.go

@@ -6,6 +6,7 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"time"
 
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -15,6 +16,11 @@ import (
 	msgraph_errors "github.com/Azure/ARO-RP/pkg/util/graph/graphsdk/models/odataerrors"
 )
 
+const (
+	GraphApiMaxRetries    int           = 5
+	GraphApiRetryInterval time.Duration = 5 * time.Second
+)
+
 func (c *Cluster) createApplication(ctx context.Context, displayName string) (string, string, error) {
 	appBody := msgraph_models.NewApplication()
 	appBody.SetDisplayName(&displayName)
@@ -35,7 +41,31 @@ func (c *Cluster) createApplication(ctx context.Context, displayName string) (st
 	// ByApplicationId is confusingly named, but it refers to
 	// the application's Object ID, not to the Application ID.
 	// https://learn.microsoft.com/en-us/graph/api/application-addpassword?view=graph-rest-1.0&tabs=http#http-request
-	pwResult, err := c.spGraphClient.Applications().ByApplicationId(id).AddPassword().Post(ctx, pwCredentialRequestBody, nil)
+
+	// retry loop, due to eventual consistency, the application we just created might not be found when queried immediately, only after a retry
+	// check if returned error is 404 not found, if so, retry the operation
+	numRetries := 0
+	var pwResult msgraph_models.PasswordCredentialable
+	err = wait.PollUntilWithContext(ctx, GraphApiRetryInterval, func(ctx context.Context) (done bool, err error) {
+		pwResult, err = c.spGraphClient.Applications().ByApplicationId(id).AddPassword().Post(ctx, pwCredentialRequestBody, nil)
+		if err == nil {
+			return true, nil
+		}
+
+		mainErr, ok := err.(*msgraph_errors.ODataError)
+		// some unknown error, bubble it up to caller
+		if !ok || mainErr.GetStatusCode() != http.StatusNotFound {
+			return false, err
+		}
+
+		// Check if we already tried too often
+		numRetries++
+		if numRetries > GraphApiMaxRetries {
+			return false, mainErr
+		}
+
+		return false, nil
+	})
 	if err != nil {
 		return "", "", err
 	}