Skip to content

Commit

Permalink
Add e2e tests for format options
Browse files Browse the repository at this point in the history
  • Loading branch information
AndrewSirenko committed Oct 4, 2023
1 parent 8f246ed commit f978ff4
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 27 deletions.
7 changes: 6 additions & 1 deletion tests/e2e/README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
## E2E Testing
E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc.
E2E test verifies the functionality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc.

### Requirements
1. AWS credential is [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
Expand All @@ -10,6 +10,11 @@ E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernet
1. sed
1. Golang 1.11+

### Running a specific test
1. Make sure you have a cluster up with ebs csi driver deployed
2. Current directory is `/aws-ebs-csi-driver/tests/e2e`
3. Run `'ginkgo run --focus='should create multiple PV objects, bind to PVCs and attach all to a single pod'`

### Notes
Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped.

Expand Down
6 changes: 5 additions & 1 deletion tests/e2e/driver/ebs_csi_driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (d *ebsCSIDriver) GetPersistentVolume(volumeID string, fsType string, size
}

// GetParameters returns the parameters specific for this driver
func GetParameters(volumeType string, fsType string, encrypted bool) map[string]string {
func GetParameters(volumeType string, fsType string, encrypted bool, additionalParameters map[string]string) map[string]string {
parameters := map[string]string{
"type": volumeType,
"csi.storage.k8s.io/fstype": fsType,
Expand All @@ -120,6 +120,10 @@ func GetParameters(volumeType string, fsType string, encrypted bool) map[string]
if encrypted {
parameters[ebscsidriver.EncryptedKey] = True
}
for k, v := range additionalParameters {
parameters[k] = v
}

return parameters
}

Expand Down
89 changes: 89 additions & 0 deletions tests/e2e/format_options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"fmt"
ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites"
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
admissionapi "k8s.io/pod-security-admission/api"
)

var (
testedFsTypes = []string{ebscsidriver.FSTypeExt4}

formatOptionTests = []testsuites.FormatOptionTest{
{
CreateVolumeParameterKey: ebscsidriver.BlockSizeKey,
CreateVolumeParameterValue: "1024",
},
{
CreateVolumeParameterKey: ebscsidriver.INodeSizeKey,
CreateVolumeParameterValue: "512",
},
{
CreateVolumeParameterKey: ebscsidriver.BytesPerINodeKey,
CreateVolumeParameterValue: "8192",
},
{
CreateVolumeParameterKey: ebscsidriver.NumberOfINodesKey,
CreateVolumeParameterValue: "200192",
},
}
)

var _ = Describe("[ebs-csi-e2e] [single-az] [format-options] Formatting a volume", func() {
f := framework.NewDefaultFramework("ebs")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged

var (
cs clientset.Interface
ns *v1.Namespace
ebsDriver driver.PVTestDriver
)

BeforeEach(func() {
cs = f.ClientSet
ns = f.Namespace
ebsDriver = driver.InitEbsCSIDriver()
})

for _, fsType := range testedFsTypes {
Context(fmt.Sprintf("with an %s filesystem", fsType), func() {
for _, formatOptionTestCase := range formatOptionTests {
formatOptionTestCase := formatOptionTestCase // Go trap
if fsTypeDoesNotSupportFormatOptionParameter(fsType, formatOptionTestCase.CreateVolumeParameterKey) {
continue
}

Context(fmt.Sprintf("with a custom %s parameter", formatOptionTestCase.CreateVolumeParameterKey), func() {
It("successfully mounts and is resizable", func() {
formatOptionTestCase.Run(cs, ns, ebsDriver, fsType)
})
})
}
})
}
})

func fsTypeDoesNotSupportFormatOptionParameter(fsType string, createVolumeParameterKey string) bool {
_, paramNotSupported := ebscsidriver.FileSystemConfigs[fsType].NotSupportedParams[createVolumeParameterKey]
return paramNotSupported
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,25 +45,7 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface,
tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
defer tpvc.Cleanup()

pvcName := tpvc.persistentVolumeClaim.Name
pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{})
By(fmt.Sprintf("Get pvc name: %v", pvc.Name))
originalSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(1))
originalSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = originalSize

By("resizing the pvc")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err))
}
updatedSize := updatedPvc.Spec.Resources.Requests["storage"]

By("checking the resizing PV result")
error := WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
framework.ExpectNoError(error)
ResizeTestPvc(client, namespace, tpvc, 1)

By("Validate volume can be attached")
tpod := NewTestPod(client, namespace, t.Pod.Cmd)
Expand Down Expand Up @@ -92,3 +74,28 @@ func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, d
}
return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName)
}

// ResizeTestPvc increases size of given persistent volume claim by `sizeIncreaseGi` Gigabytes
func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedPvc *v1.PersistentVolumeClaim, updatedSize resource.Quantity) {
By(fmt.Sprintf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name))
pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{})

originalSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(sizeIncreaseGi))
originalSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = originalSize

By("resizing the pvc")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err))
}
updatedSize = updatedPvc.Spec.Resources.Requests["storage"]

By("checking the resizing PV result")
err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
framework.ExpectNoError(err)

return
}
95 changes: 95 additions & 0 deletions tests/e2e/testsuites/format_options_tester.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import (
"fmt"
awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
)

// FormatOptionTest will provision required StorageClass(es), PVC(s) and Pod(s) TODO
// Waiting for the PV provisioner to create a new PV
// Update pvc storage size
// Note: This does
type FormatOptionTest struct {
CreateVolumeParameterKey string
CreateVolumeParameterValue string
}

const (
volumeSizeIncreaseAmtGi = 1
volumeMountPath = "/mnt/test-format-option" // TODO should I keep this as mnt/test-1, and refactor to be `DefaultMountPath` globally in testsuites?
)

var (
podCmdWriteToVolume = fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath) // TODO Debt: All the dynamic provisioning tests use this same cmd. Should we refactor out into exported constant?
)

func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespace, ebsDriver driver.PVTestDriver, fsType string) {
By("setting up pvc with custom format option")
volumeDetails := createFormatOptionVolumeDetails(fsType, volumeMountPath, t)
testPvc, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(client, namespace, ebsDriver)
defer testPvc.Cleanup()

By("deploying pod with custom format option")
getFsInfoTestPod := createPodWithVolume(client, namespace, podCmdWriteToVolume, testPvc, volumeDetails)
defer getFsInfoTestPod.Cleanup()
getFsInfoTestPod.WaitForSuccess()

By("testing that pvc is able to be resized")
ResizeTestPvc(client, namespace, testPvc, volumeSizeIncreaseAmtGi)

By("validating resized pvc by deploying new pod")
resizeTestPod := createPodWithVolume(client, namespace, podCmdWriteToVolume, testPvc, volumeDetails)
defer resizeTestPod.Cleanup()

By("confirming new pod can write to resized volume")
resizeTestPod.WaitForSuccess()
}

// TODO should we improve this across e2e tests via builder design pattern? Or is that not go-like?
func createFormatOptionVolumeDetails(fsType string, volumeMountPath string, t *FormatOptionTest) *VolumeDetails {
allowVolumeExpansion := true

volume := VolumeDetails{
VolumeType: awscloud.VolumeTypeGP2,
FSType: fsType,
MountOptions: []string{"rw"},
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeMount: VolumeMountDetails{
NameGenerate: "test-volume-format-option",
MountPathGenerate: volumeMountPath,
},
AllowVolumeExpansion: &allowVolumeExpansion,
AdditionalParameters: map[string]string{
t.CreateVolumeParameterKey: t.CreateVolumeParameterValue,
},
}

return &volume
}

// TODO putting this in function may be overkill? In an ideal world we refactor out TestEverything objects so testPod.SetupVolume isn't gross.
func createPodWithVolume(client clientset.Interface, namespace *v1.Namespace, cmd string, testPvc *TestPersistentVolumeClaim, volumeDetails *VolumeDetails) *TestPod {
testPod := NewTestPod(client, namespace, cmd)
testPod.SetupVolume(testPvc.persistentVolumeClaim, volumeDetails.VolumeMount.NameGenerate, volumeDetails.VolumeMount.MountPathGenerate, volumeDetails.VolumeMount.ReadOnly)
testPod.Create()

return testPod
}
11 changes: 5 additions & 6 deletions tests/e2e/testsuites/specs.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,9 @@ type VolumeDetails struct {
VolumeMode VolumeMode
VolumeMount VolumeMountDetails
VolumeDevice VolumeDeviceDetails
// Optional, used with pre-provisioned volumes
VolumeID string
// Optional, used with PVCs created from snapshots
DataSource *DataSource
VolumeID string // Optional, used with pre-provisioned volumes
DataSource *DataSource // Optional, used with PVCs created from snapshots
AdditionalParameters map[string]string // Optional, used when testing formatting options
}

type VolumeMode int
Expand Down Expand Up @@ -121,7 +120,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
volume := pod.Volumes[0]
By("setting up the StorageClass")

storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
Expand All @@ -141,7 +140,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) {
cleanupFuncs := make([]func(), 0)
By("setting up the StorageClass")
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
Expand Down

0 comments on commit f978ff4

Please sign in to comment.