cloud storage dedup

.github/workflows/int-test-azure-workflow.yml

@@ -9,6 +9,7 @@ on:
     branches:
       - develop
       - main
+      - CSPL-4601-rebased
     paths-ignore:
       - 'docs/**'
       - '*.md'
@@ -108,6 +109,17 @@ jobs:
         inlineScript: |
           az aks update -n ${{ env.TEST_CLUSTER_NAME }} -g ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} --attach-acr ${{ secrets.AZURE_CONTAINER_REGISTRY }}
   int-tests-appframeworkazure:
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          [
+            masterappframeworkc3,
+            managerappframeworkc3,
+            masterappframeworkm4,
+            managerappframeworkm4,
+            appframeworksS1,
+          ]
     runs-on: ubuntu-latest
     needs: setup-aks-cluster
     env:
@@ -117,7 +129,7 @@ jobs:
       SPLUNK_ENTERPRISE_RELEASE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}
       SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator
       SPLUNK_OPERATOR_IMAGE_FILENAME: splunk-operator
-      TEST_FOCUS: azure_sanity
+      TEST_FOCUS: "${{ matrix.test }}"
       # This regex matches any string not containing integration keyword
       TEST_TO_SKIP: "^(?:[^i]+|i(?:$|[^n]|n(?:$|[^t]|t(?:$|[^e]|e(?:$|[^g]|g(?:$|[^r]|r(?:$|[^a]|a(?:$|[^t]|t(?:$|[^i]|i(?:$|[^o]|o(?:$|[^n])))))))))))*$"
       TEST_CLUSTER_PLATFORM: azure
@@ -134,7 +146,6 @@ jobs:
       AZURE_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.AZURE_ACR_LOGIN_SERVER }}
       AZURE_REGION: ${{ secrets.AZURE_REGION }}
       CLUSTER_WIDE: "true"
-      # AZURE_MANAGED_ID_ENABLED: "${{ matrix.auth_method_managed_id }}"
       AZURE_MANAGED_ID_ENABLED: "false"
     steps:
       # Need this because apps are downloaded from S3.
@@ -195,34 +206,50 @@ jobs:
         run: |
           docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.AZURE_ACR_LOGIN_SERVER }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }}
           docker push ${{ secrets.AZURE_ACR_LOGIN_SERVER }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }}
-      - name: Azure Kubernetes set context
-        uses: Azure/aks-set-context@v1
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-          resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
-          cluster-name: ${{ env.TEST_CLUSTER_NAME }}
+      - name: Get AKS credentials
+        run: |
+          az aks get-credentials --resource-group ${{ secrets.AZURE_RESOURCE_GROUP_NAME }} --name ${{ env.TEST_CLUSTER_NAME }} --admin --overwrite-existing
+      - name: Setup long-lived service account auth
+        run: |
+          kubectl create serviceaccount ci-test-runner -n kube-system 2>/dev/null || true
+          kubectl create clusterrolebinding ci-test-runner-admin \
+            --clusterrole=cluster-admin \
+            --serviceaccount=kube-system:ci-test-runner 2>/dev/null || true
+
+          cat <<'EOF' | kubectl apply -f -
+          apiVersion: v1
+          kind: Secret
+          metadata:
+            name: ci-test-runner-token
+            namespace: kube-system
+            annotations:
+              kubernetes.io/service-account.name: ci-test-runner
+          type: kubernetes.io/service-account-token
+          EOF
+
+          echo "Waiting for service account token..."
+          TOKEN=""
+          for i in $(seq 1 30); do
+            TOKEN=$(kubectl get secret ci-test-runner-token -n kube-system -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null)
+            if [ -n "${TOKEN}" ]; then break; fi
+            sleep 2
+          done
+
+          if [ -n "${TOKEN}" ]; then
+            kubectl config set-credentials ci-test-runner --token="${TOKEN}"
+            kubectl config set-context --current --user=ci-test-runner
+            echo "Switched kubeconfig to long-lived service account token"
+          else
+            echo "WARNING: Could not create SA token, keeping admin cert auth"
+          fi
+          kubectl cluster-info
       - name: install metric server
-        uses: Azure/aks-set-context@v1
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-          resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
-          cluster-name: ${{ env.TEST_CLUSTER_NAME }}
-          inlineScript: |
-            curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
-            kubectl replace --force -f components.yaml || kubectl apply -f components.yaml
+        run: |
+          curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
+          kubectl replace --force -f components.yaml || kubectl apply -f components.yaml
       - name: install k8s dashboard
-        uses: Azure/aks-set-context@v1
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-          resource-group: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
-          cluster-name: ${{ env.TEST_CLUSTER_NAME }}
-          inlineScript: |
-            kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml
-      - name: Setup Kustomize
         run: |
-          sudo snap install kustomize
-          mkdir -p ./bin
-          cp /snap/bin/kustomize ./bin/kustomize
+          kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml
       - name: Run Integration test
         timeout-minutes: 240
         env:
@@ -238,20 +265,20 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v6
         with:
-          name: "splunk-pods-logs--artifacts-appframeworkazure"
+          name: "splunk-pods-logs--artifacts-${{ matrix.test }}"
           path: "/tmp/pod_logs/**"
       - name: Upload Integration Test Results
         if: always()
         uses: actions/upload-artifact@v6
         with:
-          name: "test-report-integration-azure"
+          name: "test-report-integration-azure-${{ matrix.test }}"
           path: report-junit*.xml
       - name: Publish Integration Test Report
         uses: mikepenz/action-junit-report@v6
         if: always()
         with:
           report_paths: 'report-junit*.xml'
-          check_name: 'Integration Test Results (Azure)'
+          check_name: 'Integration Test Results (Azure) - ${{ matrix.test }}'
           detailed_summary: true
       - name: Cleanup Test Case artifacts
         if: ${{ always() }}

.github/workflows/int-test-gcp-workflow.yml

@@ -9,6 +9,7 @@ on:
     branches:
       - develop
       - main
+      - CSPL-4601-rebased
     paths-ignore:
       - 'docs/**'
       - '*.md'
@@ -70,11 +71,11 @@ jobs:
     strategy:
       matrix:
         test_focus:
-          - { order: 1, name: "c3_gcp_sanity" }
-          - { order: 2, name: "c3_mgr_gcp_sanity" }
-          - { order: 3, name: "m4_gcp_sanity" }
-          - { order: 4, name: "m4_mgr_gcp_sanity" }
-          - { order: 5, name: "s1_gcp_sanity" }
+          - { order: 1, name: "masterappframeworkc3" }
+          - { order: 2, name: "managerappframeworkc3" }
+          - { order: 3, name: "masterappframeworkm4" }
+          - { order: 4, name: "managerappframeworkm4" }
+          - { order: 5, name: "appframeworksS1" }
     runs-on: ubuntu-latest
     needs: build-operator-image
     env:
@@ -84,7 +85,7 @@ jobs:
       ARTIFACT_REGISTRY: ${{ secrets.GCP_ARTIFACT_REGISTRY }}
       GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
       GCP_REGION: ${{ secrets.GCP_REGION }}
-      AWS_S3_REGION: ${{ secrets.GCP_REGION }}
+      GCP_STORAGE_REGION: ${{ secrets.GCP_REGION }}
       GCP_ZONE: ${{ secrets.GCP_ZONE }}
       GCP_NETWORK: default  # Adjust if using a custom network
       GCP_SUBNETWORK: default  # Adjust if using a custom subnetwork
@@ -112,7 +113,7 @@ jobs:
       GCP_TEST_CONTAINER: ${{ secrets.GCP_TEST_CONTAINER}}
       GCP_INDEXES_CONTAINER: ${{ secrets.GCP_INDEXES_CONTAINER}}
       ECR_REPOSITORY: ${{ secrets.GCP_ARTIFACT_REGISTRY }}
-      GCP_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.AZURE_ACR_LOGIN_SERVER }}
+      GCP_CONTAINER_REGISTRY_LOGIN_SERVER: ${{ secrets.GCP_ARTIFACT_REGISTRY }}
     steps:
       - name: Set Test Cluster Name
         run: |
@@ -136,46 +137,28 @@ jobs:
         uses: google-github-actions/setup-gcloud@v1
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
-          install_components: 'kubectl'
+          install_components: 'kubectl,gke-gcloud-auth-plugin'
 
       - name: Set GCP Project
         run: |
           gcloud config set project ${{ env.GCP_PROJECT_ID }}
+          echo "USE_GKE_GCLOUD_AUTH_PLUGIN=True" >> $GITHUB_ENV
 
       - name: Create GKE Cluster
         run: |
           export EKS_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }}
           export GKE_CLUSTER_K8_VERSION=${{ steps.dotenv.outputs.EKS_CLUSTER_K8_VERSION }}
           make cluster-up
 
-      - name: Get Kubernetes Credentials
-        run: |
-          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }}
-
       - name: Allow Pulling from Artifact Registry
         run: |
           gcloud auth configure-docker ${{ secrets.GCP_ARTIFACT_REGISTRY }}
 
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v1
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          install_components: 'kubectl'
-
       - name: Change Splunk Enterprise Image on Main Branches
         if: github.ref == 'refs/heads/main'
         run: |
           echo "SPLUNK_ENTERPRISE_IMAGE=${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}" >> $GITHUB_ENV
 
-      - name: Authenticate to GCP
-        uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-
-      - name: Set GCP Project
-        run: |
-          gcloud config set project ${{ env.GCP_PROJECT_ID }}
-
       - name: Install Kubectl
         uses: azure/setup-kubectl@v3
         with:
@@ -224,16 +207,6 @@ jobs:
           docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }}
           docker push ${{ secrets.GCP_ARTIFACT_REGISTRY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }}
 
-      - name: Get Kubernetes Credentials
-        run: |
-          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --zone ${{ env.GCP_ZONE }} --project ${{ env.GCP_PROJECT_ID }}
-
-      - name: Get GKE Credentials
-        uses: google-github-actions/get-gke-credentials@v1
-        with:
-          cluster_name: ${{ env.CLUSTER_NAME }}
-          location: ${{ env.GCP_ZONE }}
-
       - name: Install Metrics Server
         run: |
           curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
@@ -245,7 +218,7 @@ jobs:
 
       - name: Setup Kustomize
         run: |
-          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"  | bash
+          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 5.4.3
           sudo mv kustomize /usr/local/bin/
 
       - name: Verify kubectl Configuration

Makefile

@@ -236,10 +236,9 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION}
 
 KUSTOMIZE = $(LOCALBIN)/kustomize
-KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
 $(KUSTOMIZE): $(LOCALBIN)
-	test -s $(LOCALBIN)/kustomize || curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,${KUSTOMIZE_VERSION}) $(LOCALBIN)
+	test -s $(LOCALBIN)/kustomize || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kustomize/kustomize/v5@${KUSTOMIZE_VERSION}
 
 ENVTEST = $(LOCALBIN)/setup-envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.

test/appframework_aws/m4/appframework_aws_test.go

@@ -55,8 +55,10 @@ var _ = Describe("m4appfw test", func() {
 	BeforeEach(func() {
 		var err error
 		name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3))
+		fmt.Printf("[DEBUG] M4 BeforeEach starting, name=%s\n", name)
 		testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name)
 		Expect(err).To(Succeed(), "Unable to create testcaseenv")
+		fmt.Printf("[DEBUG] M4 BeforeEach testcaseenv created, namespace=%s\n", testcaseEnvInst.GetName())
 		deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3))
 		Expect(err).To(Succeed(), "Unable to create deployment")
 
@@ -68,6 +70,7 @@ var _ = Describe("m4appfw test", func() {
 		s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4)
 		appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3)
 		appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3)
+		fmt.Printf("[DEBUG] M4 BeforeEach complete, deployment=%s\n", deployment.GetName())
 	})
 
 	AfterEach(func() {
@@ -1106,11 +1109,13 @@ var _ = Describe("m4appfw test", func() {
 			testcaseEnvInst.Log.Info("Get config map for triggering manual update")
 			config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName())
 			Expect(err).To(Succeed(), "Unable to get config map for manual poll")
+			testcaseEnvInst.Log.Info("[DEBUG] ConfigMap state before CM manual poll trigger", "ClusterMaster", config.Data["ClusterMaster"], "SearchHeadCluster", config.Data["SearchHeadCluster"], "MonitoringConsole", config.Data["MonitoringConsole"])
 
-			testcaseEnvInst.Log.Info("Modify config map to trigger manual update")
+			testcaseEnvInst.Log.Info("Modify config map to trigger manual update for ClusterMaster")
 			config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1)
 			err = deployment.UpdateCR(ctx, config)
 			Expect(err).To(Succeed(), "Unable to update config map")
+			testcaseEnvInst.Log.Info("[DEBUG] ConfigMap updated for CM manual poll", "ClusterMaster", config.Data["ClusterMaster"])
 
 			// Ensure that the Cluster Master goes to Ready phase
 			testcaseEnvInst.VerifyClusterMasterReady(ctx, deployment)
@@ -1121,29 +1126,33 @@ var _ = Describe("m4appfw test", func() {
 			// Ensure Indexer cluster configured as multisite
 			testcaseEnvInst.VerifyIndexerClusterMultisiteStatus(ctx, deployment, siteCount)
 
-			testcaseEnvInst.Log.Info("Get config map for triggering manual update")
+			testcaseEnvInst.Log.Info("Get config map for triggering manual update for SHC")
 			config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName())
 			Expect(err).To(Succeed(), "Unable to get config map for manual poll")
+			testcaseEnvInst.Log.Info("[DEBUG] ConfigMap state before SHC manual poll trigger", "ClusterMaster", config.Data["ClusterMaster"], "SearchHeadCluster", config.Data["SearchHeadCluster"], "MonitoringConsole", config.Data["MonitoringConsole"])
 
-			testcaseEnvInst.Log.Info("Modify config map to trigger manual update")
+			testcaseEnvInst.Log.Info("Modify config map to trigger manual update for SearchHeadCluster")
 			config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1)
 			err = deployment.UpdateCR(ctx, config)
 			Expect(err).To(Succeed(), "Unable to update config map")
+			testcaseEnvInst.Log.Info("[DEBUG] ConfigMap updated for SHC manual poll", "SearchHeadCluster", config.Data["SearchHeadCluster"])
 
 			// Ensure Search Head Cluster go to Ready phase
 			testcaseEnvInst.VerifySearchHeadClusterReady(ctx, deployment)
 
 			// Verify RF SF is met
 			testcaseEnvInst.VerifyRFSFMet(ctx, deployment)
 
-			testcaseEnvInst.Log.Info("Get config map for triggering manual update")
+			testcaseEnvInst.Log.Info("Get config map for triggering manual update for MC")
 			config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName())
 			Expect(err).To(Succeed(), "Unable to get config map for manual poll")
+			testcaseEnvInst.Log.Info("[DEBUG] ConfigMap state before MC manual poll trigger", "ClusterMaster", config.Data["ClusterMaster"], "SearchHeadCluster", config.Data["SearchHeadCluster"], "MonitoringConsole", config.Data["MonitoringConsole"])
 
-			testcaseEnvInst.Log.Info("Modify config map to trigger manual update")
+			testcaseEnvInst.Log.Info("Modify config map to trigger manual update for MonitoringConsole")
 			config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1)
 			err = deployment.UpdateCR(ctx, config)
 			Expect(err).To(Succeed(), "Unable to update config map")
+			testcaseEnvInst.Log.Info("[DEBUG] ConfigMap updated for MC manual poll", "MonitoringConsole", config.Data["MonitoringConsole"])
 
 			// Wait for Monitoring Console to reach Ready phase
 			err = testcaseEnvInst.WaitForMonitoringConsolePhase(ctx, deployment, testcaseEnvInst.GetName(), mc.Name, enterpriseApi.PhaseReady, 2*time.Minute)
@@ -1164,6 +1173,11 @@ var _ = Describe("m4appfw test", func() {
 			Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete")
 
 			// ############ VERIFY APPS UPDATED TO V2 #############
+			testcaseEnvInst.Log.Info("[DEBUG] Starting V2 verification", "ClusterMasterBundleHash", ClusterMasterBundleHash, "IdxcPods", idxcPodNames, "ShcPods", shcPodNames)
+			configDbg, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName())
+			if configDbg != nil {
+				testcaseEnvInst.Log.Info("[DEBUG] Final ConfigMap state before V2 verification", "ClusterMaster", configDbg.Data["ClusterMaster"], "SearchHeadCluster", configDbg.Data["SearchHeadCluster"], "MonitoringConsole", configDbg.Data["MonitoringConsole"])
+			}
 			appVersion = "V2"
 			cmAppSourceInfo.CrAppVersion = appVersion
 			cmAppSourceInfo.CrAppList = appListV2

test/appframework_aws/m4/manager_appframework_test.go

@@ -54,8 +54,10 @@ var _ = Describe("m4appfw test", func() {
 	BeforeEach(func() {
 		var err error
 		name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3))
+		fmt.Printf("[DEBUG] M4-manager BeforeEach starting, name=%s\n", name)
 		testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name)
 		Expect(err).To(Succeed(), "Unable to create testcaseenv")
+		fmt.Printf("[DEBUG] M4-manager BeforeEach testcaseenv created, namespace=%s\n", testcaseEnvInst.GetName())
 		deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3))
 		Expect(err).To(Succeed(), "Unable to create deployment")
 
@@ -67,6 +69,7 @@ var _ = Describe("m4appfw test", func() {
 		s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4)
 		appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3)
 		appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3)
+		fmt.Printf("[DEBUG] M4-manager BeforeEach complete, deployment=%s\n", deployment.GetName())
 	})
 
 	AfterEach(func() {

test/deploy-eks-cluster.sh

@@ -140,8 +140,6 @@ function createCluster() {
       return 1
   fi
 
-
-  # Login to ECR registry so images can be push and pull from later whe
   # Output
   echo "EKS cluster nodes:"
   eksctl get cluster --name=${TEST_CLUSTER_NAME}