Adding E2E tests, examples and documentation

charts/aws-efs-csi-driver/values.yaml

@@ -60,7 +60,7 @@ controller:
     # environment: prod
     # region: us-east-1
   # Enable if you want the controller to also delete the
-  # path on efs when deleteing an access point
+  # path on efs when deleting an EFS access point
   deleteAccessPointRootDir: false
   # Enable if you want the controller to delete any directories it also provisions
   deleteProvisionedDir: false

examples/kubernetes/dynamic_provisioning/README.md → examples/kubernetes/dynamic_provisioning/access_points/README.md

@@ -51,7 +51,7 @@ This example requires Kubernetes 1.17 or later and a driver version of 1.2.0 or
    1. Download a manifest that deploys a Pod and a PVC.
 
       ```sh
-      curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/pod.yaml
+      curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/master/examples/kubernetes/dynamic_provisioning/access_points/specs/pod.yaml
       ```
 
    1. Deploy the Pod with a sample app and the PVC used by the Pod.

examples/kubernetes/dynamic_provisioning/directories/README.md

@@ -0,0 +1,152 @@
+## Dynamic Provisioning
+**Important**  
+You can't use dynamic provisioning with Fargate nodes.
+
+This example shows how to create a dynamically provisioned volume created through a directory on the file system and a Persistent Volume Claim (PVC) and consume it from a pod.
+
+**Prerequisite**  
+This example requires Kubernetes 1.17 or later and a driver version of 1.5.x or later.
+
+1. Create a storage class for Amazon EFS.
+
+    1. Retrieve your Amazon EFS file system ID. You can find this in the Amazon EFS console, or use the following AWS CLI command.
+
+       ```sh
+       aws efs describe-file-systems --query "FileSystems[*].FileSystemId" --output text
+       ```
+
+       The example output is as follows.
+
+       ```
+       fs-582a03f3
+       ```
+
+    2. Download a `StorageClass` manifest for Amazon EFS.
+
+       ```sh
+       curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/master/examples/kubernetes/dynamic_provisioning/directories/specs/storageclass.yaml
+       ```
+
+    3. Edit [the file](./specs/storageclass.yaml). Find the following line, and replace the value for `fileSystemId` with your file system ID.
+
+       ```
+       fileSystemId: fs-582a03f3
+       ```
+
+       Modify the other values as needed:
+        * `fileSystemId` - The file system under which the access point is created.
+        * `directoryPerms` - The directory permissions of the root directory created by the access point.
+        * `gidRangeStart` (Optional) - The starting range of the Posix group ID to be applied onto the root directory of the access point. The default value is `50000`.
+        * `gidRangeEnd` (Optional) - The ending range of the Posix group ID. The default value is `7000000`.
+        * `basePath` (Optional) - The path on the file system under which the access point root directory is created. If the path isn't provided, the access points root directory is created under the root of the file system.
+
+    4. Deploy the storage class.
+
+       ```sh
+       kubectl apply -f storageclass.yaml
+       ```
+
+2. Test automatic provisioning by deploying a Pod that makes use of the PVC:
+
+    1. Download a manifest that deploys a Pod and a PVC.
+
+       ```sh
+       curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/master/examples/kubernetes/dynamic_provisioning/specs/pod.yaml
+       ```
+
+    2. Deploy the Pod with a sample app and the PVC used by the Pod.
+
+       ```sh
+       kubectl apply -f pod.yaml
+       ```
+
+3. Determine the names of the Pods running the controller.
+
+   ```sh
+   kubectl get pods -n kube-system | grep efs-csi-controller
+   ```
+
+   The example output is as follows.
+
+   ```
+   efs-csi-controller-74ccf9f566-q5989   3/3     Running   0          40m
+   efs-csi-controller-74ccf9f566-wswg9   3/3     Running   0          40m
+   ```
+
+4. After few seconds, you can observe the controller picking up the change \(edited for readability\). Replace `74ccf9f566-q5989` with a value from one of the Pods in your output from the previous command.
+
+   ```sh
+   kubectl logs efs-csi-controller-74ccf9f566-q5989 \
+       -n kube-system \
+       -c csi-provisioner \
+       --tail 10
+   ```
+
+   The example output is as follows.
+
+   ```
+   [...]
+   1 controller.go:737] successfully created PV pvc-5983ffec-96cf-40c1-9cd6-e5686ca84eca for PVC efs-claim...
+   ```
+
+   If you don't see the previous output, run the previous command using one of the other controller Pods.
+
+5. Confirm that a persistent volume was created with a status of `Bound` to a `PersistentVolumeClaim`:
+
+   ```sh
+   kubectl get pv
+   ```
+
+   The example output is as follows.
+
+   ```
+   NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM               STORAGECLASS   REASON   AGE
+   pvc-5983ffec-96cf-40c1-9cd6-e5686ca84eca   20Gi       RWX            Delete           Bound    default/efs-claim   efs-sc                  7m57s
+   ```
+
+6. View details about the `PersistentVolumeClaim` that was created.
+
+   ```sh
+   kubectl get pvc
+   ```
+
+   The example output is as follows.
+
+   ```
+   NAME        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+   efs-claim   Bound    pvc-5983ffec-96cf-40c1-9cd6-e5686ca84eca   20Gi       RWX            efs-sc         9m7s
+   ```
+
+7. View the sample app Pod's status until the `STATUS` becomes `Running`.
+
+   ```sh
+   kubectl get pods -o wide
+   ```
+
+   The example output is as follows.
+
+   ```
+   NAME          READY   STATUS    RESTARTS   AGE   IP               NODE                                             NOMINATED NODE   READINESS GATES
+   efs-app       1/1     Running   0          10m   192.168.78.156   ip-192-168-73-191.region-code.compute.internal   <none>           <none>
+   ```
+**Note**  
+If a Pod doesn't have an IP address listed, make sure that you added a mount target for the subnet that your node is in \(as described at the end of [Create an Amazon EFS file system](#efs-create-filesystem)\). Otherwise the Pod won't leave `ContainerCreating` status. When an IP address is listed, it may take a few minutes for a Pod to reach the `Running` status.
+
+1. Confirm that the data is written to the volume.
+
+   ```sh
+   kubectl exec efs-app -- bash -c "cat data/out"
+   ```
+
+   The example output is as follows.
+
+   ```
+   [...]
+   Tue Mar 23 14:29:16 UTC 2021
+   Tue Mar 23 14:29:21 UTC 2021
+   Tue Mar 23 14:29:26 UTC 2021
+   Tue Mar 23 14:29:31 UTC 2021
+   [...]
+   ```
+
+2. \(Optional\) Terminate the Amazon EKS node that your Pod is running on and wait for the Pod to be re\-scheduled. Alternately, you can delete the Pod and redeploy it. Complete the previous step again, confirming that the output includes the previous output.

examples/kubernetes/dynamic_provisioning/directories/specs/pod.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: efs-claim
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: efs-sc
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: efs-app
+spec:
+  containers:
+    - name: app
+      image: centos
+      command: ["/bin/sh"]
+      args: ["-c", "while true; do echo $(date -u) >> /data/out; sleep 5; done"]
+      volumeMounts:
+        - name: persistent-storage
+          mountPath: /data
+  volumes:
+    - name: persistent-storage
+      persistentVolumeClaim:
+        claimName: efs-claim

examples/kubernetes/dynamic_provisioning/directories/specs/storageclass.yaml

@@ -0,0 +1,12 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: efs-sc
+provisioner: efs.csi.aws.com
+parameters:
+  provisioningMode: efs-dir
+  fileSystemId: fs-92107410
+  directoryPerms: "700"
+  gidRangeStart: "1000" # optional
+  gidRangeEnd: "2000" # optional
+  basePath: "/dynamic_provisioning" # optional

hack/values.yaml

@@ -1,8 +1,9 @@
 controller:
   create: true
   logLevel: 5
+  serviceAccount:
+    create: true
+  deleteProvisionedDir: true
 node:
   logLevel: 5
-serviceAccount:
-  controller:
-    create: true
+

hack/values_eksctl.yaml

@@ -2,6 +2,7 @@ controller:
   logLevel: 5
   serviceAccount:
     create: false # let eksctl create it
+  deleteProvisionedDir: true
 node:
   logLevel: 5
   serviceAccount:

pkg/driver/directory_provisioner.go

@@ -61,7 +61,7 @@ func (d DirectoryProvisioner) Provision(ctx context.Context, req *csi.CreateVolu
 	}
 
 	rootDirName := req.Name
-	provisionedPath = basePath + "/" + rootDirName
+	provisionedPath = path.Join(basePath, rootDirName)
 
 	klog.V(5).Infof("Provisioning directory at path %s", provisionedPath)
 
@@ -82,6 +82,13 @@ func (d DirectoryProvisioner) Provision(ctx context.Context, req *csi.CreateVolu
 		return nil, status.Errorf(codes.Internal, "Could not provision directory: %v", err)
 	}
 
+	// Check the permissions that actually got created
+	actualPerms, err := d.osClient.GetPerms(provisionedDirectory)
+	if err != nil {
+		klog.V(5).Infof("Could not load file info for '%s'", provisionedDirectory)
+	}
+	klog.V(5).Infof("Permissions of folder '%s' are '%s'", provisionedDirectory, actualPerms)
+
 	err = d.mounter.Unmount(target)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "Could not unmount %q: %v", target, err)
@@ -103,6 +110,7 @@ func (d DirectoryProvisioner) Delete(ctx context.Context, req *csi.DeleteVolumeR
 		return nil
 	}
 	fileSystemId, subpath, _, _ := parseVolumeId(req.GetVolumeId())
+	klog.V(5).Infof("Running delete for EFS %s at subpath %s", fileSystemId, subpath)
 
 	localCloud, roleArn, err := getCloud(d.cloud, req.GetSecrets())
 	if err != nil {
@@ -115,31 +123,39 @@ func (d DirectoryProvisioner) Delete(ctx context.Context, req *csi.DeleteVolumeR
 	}
 
 	target := TempMountPathPrefix + "/" + uuid.New().String()
+	klog.V(5).Infof("Making temporary directory at '%s' to temporarily mount EFS folder in", target)
 	if err := d.mounter.MakeDir(target); err != nil {
 		return status.Errorf(codes.Internal, "Could not create dir %q: %v", target, err)
 	}
 
 	defer func() {
 		// Try and unmount the directory
+		klog.V(5).Infof("Unmounting directory mounted at '%s'", target)
 		unmountErr := d.mounter.Unmount(target)
 		// If that fails then track the error but don't do anything else
 		if unmountErr != nil {
+			klog.V(5).Infof("Unmount failed at '%s'", target)
 			e = status.Errorf(codes.Internal, "Could not unmount %q: %v", target, err)
 		} else {
 			// If it is nil then it's safe to try and delete the directory as it should now be empty
+			klog.V(5).Infof("Deleting temporary directory at '%s'", target)
 			if err := d.osClient.RemoveAll(target); err != nil {
 				e = status.Errorf(codes.Internal, "Could not delete %q: %v", target, err)
 			}
 		}
 	}()
 
+	klog.V(5).Infof("Mounting EFS '%s' into temporary directory at '%s'", fileSystemId, target)
 	if err := d.mounter.Mount(fileSystemId, target, "efs", mountOptions); err != nil {
 		// If this call throws an error we're about to return anyway and the mount has failed, so it's more
 		// important we return with that information than worry about the folder not being deleted
 		_ = d.osClient.Remove(target)
 		return status.Errorf(codes.Internal, "Could not mount %q at %q: %v", fileSystemId, target, err)
 	}
-	if err := d.osClient.RemoveAll(target + subpath); err != nil {
+
+	pathToRemove := path.Join(target, subpath)
+	klog.V(5).Infof("Delete all files at %s, stored on EFS %s", pathToRemove, fileSystemId)
+	if err := d.osClient.RemoveAll(pathToRemove); err != nil {
 		return status.Errorf(codes.Internal, "Could not delete directory %q: %v", subpath, err)
 	}
 

pkg/driver/os_client.go

@@ -5,6 +5,7 @@ import "os"
 type OsClient interface {
 	MkDirAllWithPerms(path string, perms os.FileMode, uid, gid int) error
 	MkDirAllWithPermsNoOwnership(path string, perms os.FileMode) error
+	GetPerms(path string) (os.FileMode, error)
 	Remove(path string) error
 	RemoveAll(path string) error
 }
@@ -27,6 +28,10 @@ func (o *FakeOsClient) RemoveAll(_ string) error {
 	return nil
 }
 
+func (o *FakeOsClient) GetPerms(_ string) (os.FileMode, error) {
+	return 0, nil
+}
+
 type BrokenOsClient struct{}
 
 func (o *BrokenOsClient) MkDirAllWithPerms(_ string, _ os.FileMode, _, _ int) error {
@@ -45,13 +50,22 @@ func (o *BrokenOsClient) RemoveAll(_ string) error {
 	return &os.PathError{}
 }
 
+func (o *BrokenOsClient) GetPerms(path string) (os.FileMode, error) {
+	return 0, &os.PathError{}
+}
+
 type RealOsClient struct{}
 
 func (o *RealOsClient) MkDirAllWithPerms(path string, perms os.FileMode, uid, gid int) error {
 	err := os.MkdirAll(path, perms)
 	if err != nil {
 		return err
 	}
+	// Extra CHMOD guarantees we get the permissions we desire, inspite of the umask
+	err = os.Chmod(path, perms)
+	if err != nil {
+		return err
+	}
 	err = os.Chown(path, uid, gid)
 	if err != nil {
 		return err
@@ -64,6 +78,11 @@ func (o *RealOsClient) MkDirAllWithPermsNoOwnership(path string, perms os.FileMo
 	if err != nil {
 		return err
 	}
+	// Extra CHMOD guarantees we get the permissions we desire, inspite of the umask
+	err = os.Chmod(path, perms)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -74,3 +93,11 @@ func (o *RealOsClient) Remove(path string) error {
 func (o *RealOsClient) RemoveAll(path string) error {
 	return os.RemoveAll(path)
 }
+
+func (o *RealOsClient) GetPerms(path string) (os.FileMode, error) {
+	fInfo, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+	return fInfo.Mode(), nil
+}