Fix policy attachment flakes (#9558)

solo-io · May 31, 2024 · f8ea746 · f8ea746
1 parent 0637825
commit f8ea746
Show file tree

Hide file tree

Showing 11 changed files with 50 additions and 23 deletions.
diff --git a/changelog/v1.17.0-beta34/policy-attachment-flakes.yaml b/changelog/v1.17.0-beta34/policy-attachment-flakes.yaml
@@ -0,0 +1,6 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/gloo/issues/9494
+    resolvesIssue: true
+    description: >-
+      Addressed some noisy flakes in the new Kubernetes E2E tests for Policy Attachment
diff --git a/test/kubernetes/e2e/features/route_options/suite.go b/test/kubernetes/e2e/features/route_options/suite.go
@@ -142,10 +142,7 @@ func (s *testingSuite) TestConfigureInvalidRouteOptionsWithTargetRef() {
 func (s *testingSuite) TestConfigureRouteOptionsWithMultipleTargetRefManualSetup() {
 	// Manually apply our manifests so we can assert that basic rto exists before applying extra rto.
 	// This is needed because our solo-kit clients currently do not return creationTimestamp
-	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest)
-	s.NoError(err, "can apply "+setupManifest)
-
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, httproute1Manifest)
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, httproute1Manifest)
 	s.NoError(err, "can apply "+httproute1Manifest)
 
 	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, basicRtoTargetRefManifest)
@@ -188,10 +185,7 @@ func (s *testingSuite) TestConfigureRouteOptionsWithMultipleTargetRefManualSetup
 func (s *testingSuite) TestConfigureRouteOptionsWithMultipleFilterExtensionManualSetup() {
 	// Manually apply our manifests so we can assert that basic rto exists before applying extra rto.
 	// This is needed because our solo-kit clients currently do not return creationTimestamp
-	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest)
-	s.NoError(err, "can apply "+setupManifest)
-
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, httproute1MultipleExtensionsManifest)
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, httproute1MultipleExtensionsManifest)
 	s.NoError(err, "can apply "+httproute1MultipleExtensionsManifest)
 
 	// here we apply the extra manifest first so that it is older

diff --git a/test/kubernetes/e2e/features/virtualhost_options/suite.go b/test/kubernetes/e2e/features/virtualhost_options/suite.go
@@ -140,10 +140,7 @@ func (s *testingSuite) TestConfigureInvalidVirtualHostOptions() {
 func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup() {
 	// Manually apply our manifests so we can assert that basic vho exists before applying extra vho.
 	// This is needed because our solo-kit clients currently do not return creationTimestamp
-	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest)
-	s.NoError(err, "can apply "+setupManifest)
-
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, basicVhOManifest)
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, basicVhOManifest)
 	s.NoError(err, "can apply "+basicVhOManifest)
 	// Check status is accepted before moving on to apply conflicting vho
 	s.testInstallation.Assertions.EventuallyResourceStatusMatchesState(
@@ -207,10 +204,7 @@ func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup
 func (s *testingSuite) TestMultipleVirtualHostOptionsManualSetup() {
 	// Manually apply our manifests so we can assert that basic vho exists before applying extra vho.
 	// This is needed because our solo-kit clients currently do not return creationTimestamp
-	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest)
-	s.NoError(err, "can apply "+setupManifest)
-
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, basicVhOManifest)
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, basicVhOManifest)
 	s.NoError(err, "can apply "+basicVhOManifest)
 	// Check status is accepted before moving on to apply conflicting vho
 	s.testInstallation.Assertions.EventuallyResourceStatusMatchesState(

diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go
@@ -177,9 +177,14 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) {
 	// This is a work in progress
 	// The idea here is we want to accumulate ALL information about this TestInstallation into a single directory
 	// That way we can upload it in CI, or inspect it locally
-	logFile := filepath.Join(i.GeneratedFiles.FailureDir, "gloo.txt")
-	logsCmd := i.Actions.Kubectl().Command(ctx, "logs", "-n", i.Metadata.InstallNamespace, "deployments/gloo", ">", logFile)
-	_ = logsCmd.Run()
+
+	glooLogFilePath := filepath.Join(i.GeneratedFiles.FailureDir, "gloo.log")
+	glooLogFile, err := os.Create(glooLogFilePath)
+	i.Assertions.Require.NoError(err)
+	defer glooLogFile.Close()
+
+	logsCmd := i.Actions.Kubectl().Command(ctx, "logs", "-n", i.Metadata.InstallNamespace, "deployments/gloo")
+	_ = logsCmd.WithStdout(glooLogFile).WithStderr(glooLogFile).Run()
 }
 
 // GeneratedFiles is a collection of files that are generated during the execution of a set of tests

diff --git a/test/kubernetes/e2e/tests/manifests/edge-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/edge-gateway-test-helm.yaml
@@ -14,6 +14,10 @@ gateway:
   validation:
     allowWarnings: true
     alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: false

diff --git a/test/kubernetes/e2e/tests/manifests/istio-automtls-edge-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-automtls-edge-gateway-test-helm.yaml
@@ -19,6 +19,10 @@ gateway:
   validation:
     allowWarnings: true
     alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: false

diff --git a/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml
@@ -26,6 +26,10 @@ gateway:
   validation:
     allowWarnings: true
     alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: true
@@ -40,4 +44,4 @@ gatewayProxies:
     healthyPanicThreshold: 0
 # Disable discovery, not recommended for production.
 discovery:
-  enabled: false
+  enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/istio-edge-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-edge-gateway-test-helm.yaml
@@ -19,6 +19,10 @@ gateway:
   validation:
     allowWarnings: true
     alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: false

diff --git a/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml
@@ -26,6 +26,10 @@ gateway:
   validation:
     allowWarnings: true
     alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: true
@@ -40,4 +44,4 @@ gatewayProxies:
     healthyPanicThreshold: 0
 # Disable discovery, not recommended for production
 discovery:
-  enabled: false
+  enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml
@@ -23,6 +23,10 @@ gateway:
     allowWarnings: true
     # always accept regardless of validation result
     alwaysAcceptResources: true
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: true
@@ -37,4 +41,4 @@ gatewayProxies:
     healthyPanicThreshold: 0
 # Disable discovery, not recommended for production
 discovery:
-  enabled: false
+  enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml
@@ -21,6 +21,10 @@ gateway:
   validation:
     allowWarnings: true
     alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
 kubeGateway:
   # This is the field that enables the K8s Gateway Integration in Gloo Gateway
   enabled: true
@@ -35,4 +39,4 @@ gatewayProxies:
     healthyPanicThreshold: 0
 # Disable discovery, not recommended for production
 discovery:
-  enabled: false
+  enabled: false