Toplogy unit and e2e testing

Author: davidz627

pkg/gce-pd-csi-driver/controller_test.go

@@ -136,6 +136,65 @@ func TestCreateVolumeArguments(t *testing.T) {
 				Attributes:    nil,
 			},
 		},
+		{
+			name: "success with topology",
+			req: &csi.CreateVolumeRequest{
+				Name:               "test-vol",
+				CapacityRange:      stdCapRange,
+				VolumeCapabilities: stdVolCap,
+				Parameters:         map[string]string{"type": "test-type"},
+				AccessibilityRequirements: &csi.TopologyRequirement{
+					Requisite: []*csi.Topology{
+						{
+							Segments: map[string]string{"zone": "topology-zone"},
+						},
+					},
+				},
+			},
+			expVol: &csi.Volume{
+				CapacityBytes: utils.GbToBytes(20),
+				Id:            "topology-zone/test-vol",
+				Attributes:    nil,
+			},
+		},
+		{
+			name: "success with topology overwrite zone in params",
+			req: &csi.CreateVolumeRequest{
+				Name:               "test-vol",
+				CapacityRange:      stdCapRange,
+				VolumeCapabilities: stdVolCap,
+				Parameters:         stdParams,
+				AccessibilityRequirements: &csi.TopologyRequirement{
+					Requisite: []*csi.Topology{
+						{
+							Segments: map[string]string{"zone": "topology-zone"},
+						},
+					},
+				},
+			},
+			expVol: &csi.Volume{
+				CapacityBytes: utils.GbToBytes(20),
+				Id:            "topology-zone/test-vol",
+				Attributes:    nil,
+			},
+		},
+		{
+			name: "fail with malformed topology",
+			req: &csi.CreateVolumeRequest{
+				Name:               "test-vol",
+				CapacityRange:      stdCapRange,
+				VolumeCapabilities: stdVolCap,
+				Parameters:         stdParams,
+				AccessibilityRequirements: &csi.TopologyRequirement{
+					Requisite: []*csi.Topology{
+						{
+							Segments: map[string]string{"ooblezoners": "topology-zone"},
+						},
+					},
+				},
+			},
+			expErrCode: codes.InvalidArgument,
+		},
 	}
 
 	// Run test cases
@@ -181,7 +240,7 @@ func TestCreateVolumeArguments(t *testing.T) {
 		}
 
 		if vol.GetId() != tc.expVol.GetId() {
-			t.Fatalf("Expected volume id: %v, got: %v", vol.GetId(), tc.expVol.GetId())
+			t.Fatalf("Expected volume id: %q, got: %q", vol.GetId(), tc.expVol.GetId())
 		}
 
 		for akey, aval := range tc.expVol.GetAttributes() {

pkg/gce-pd-csi-driver/identity_test.go

@@ -59,6 +59,7 @@ func TestGetPluginCapabilities(t *testing.T) {
 	for _, capability := range resp.GetCapabilities() {
 		switch capability.GetService().GetType() {
 		case csi.PluginCapability_Service_CONTROLLER_SERVICE:
+		case csi.PluginCapability_Service_ACCESSIBILITY_CONSTRAINTS:
 		default:
 			t.Fatalf("Unknown capability: %v", capability.GetService().GetType())
 		}

test/e2e/tests/multi_zone_e2e_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
+)
+
+var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
+	BeforeEach(func() {
+		Expect(len(testInstances)).To(BeNumerically(">", 1))
+		// TODO: Check whether the instances are in different zones???
+		// I Think there should be a better way of guaranteeing this. Like a map from zone to instance for testInstances (?)
+	})
+
+	It("Should get reasonable toplogy from nodes with NodeGetInfo", func() {
+		for _, instance := range testInstances {
+			testContext, err := utils.SetupNewDriverAndClient(instance)
+			Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
+
+			resp, err := testContext.Client.NodeGetInfo()
+			Expect(err).To(BeNil())
+
+			// Get Cloud Instance
+			p, z, n := testContext.Instance.GetIdentity()
+			cloudInstance, err := computeService.Instances.Get(p, z, n).Do()
+			Expect(err).To(BeNil())
+			Expect(cloudInstance).ToNot(BeNil())
+
+			// Check topology matches
+			segments := resp.GetAccessibleTopology().GetSegments()
+			Expect(segments).ToNot(BeNil())
+
+			Expect(segments["zone"]).To(Equal(z))
+			Expect(len(segments)).To(Equal(1))
+		}
+
+	})
+
+})

test/e2e/tests/setup_e2e_test.go

@@ -16,20 +16,72 @@ package tests
 
 import (
 	"flag"
+	"fmt"
+	"math/rand"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	compute "google.golang.org/api/compute/v1"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
+	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
 )
 
 var (
-	project        = flag.String("project", "", "Project to run tests in")
-	serviceAccount = flag.String("service-account", "", "Service account to bring up instance with")
-	runInProw      = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
+	project         = flag.String("project", "", "Project to run tests in")
+	serviceAccount  = flag.String("service-account", "", "Service account to bring up instance with")
+	runInProw       = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
+	deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run")
+
+	testInstances  = []*remote.InstanceInfo{}
+	computeService *compute.Service
 )
 
 func TestE2E(t *testing.T) {
 	flag.Parse()
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests")
 }
+
+var _ = BeforeSuite(func() {
+	var err error
+	zones := []string{"us-central1-c", "us-central1-b"}
+
+	rand.Seed(time.Now().UnixNano())
+
+	computeService, err = remote.GetComputeClient()
+	Expect(err).To(BeNil())
+
+	for _, zone := range zones {
+		nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
+
+		if *runInProw {
+			*project, *serviceAccount = utils.SetupProwConfig()
+		}
+
+		Expect(*project).ToNot(BeEmpty(), "Project should not be empty")
+		Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty")
+
+		i, err := utils.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
+		Expect(err).To(BeNil())
+
+		testInstances = append(testInstances, i)
+	}
+
+})
+
+var _ = AfterSuite(func() {
+	/*
+		err := node.client.CloseConn()
+		if err != nil {
+			Logf("Failed to close the client")
+		} else {
+			Logf("Closed the client")
+	*/
+	for _, i := range testInstances {
+		if *deleteInstances {
+			i.DeleteInstance()
+		}
+	}
+})

test/e2e/tests/single_zone_e2e_test.go

@@ -20,15 +20,15 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
-	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
+	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
 
+	csi "github.com/container-storage-interface/spec/lib/go/csi/v0"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 const (
-	network        = "unix"
 	testNamePrefix = "gcepd-csi-e2e-"
 
 	defaultSizeGb    int64 = 5
@@ -37,83 +37,48 @@ const (
 	ssdDiskType            = "pd-ssd"
 )
 
-var (
-	client   *utils.CsiClient
-	instance *remote.InstanceInfo
-	//gceCloud *gce.CloudProvider
-	nodeID string
-)
-
-var _ = BeforeSuite(func() {
-	var err error
-	// TODO(dyzz): better defaults
-	nodeID = "gce-pd-csi-e2e-us-central1-c"
-	port := "2000"
-	if *runInProw {
-		*project, *serviceAccount = utils.SetupProwConfig()
-	}
-
-	Expect(*project).ToNot(BeEmpty(), "Project should not be empty")
-	Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty")
-
-	instance, err = utils.SetupInstanceAndDriver(*project, "us-central1-c", nodeID, port, *serviceAccount)
-	Expect(err).To(BeNil())
-
-	client = utils.CreateCSIClient(fmt.Sprintf("localhost:%s", port))
-})
-
-var _ = AfterSuite(func() {
-	// Close the client
-	err := client.CloseConn()
-	if err != nil {
-		Logf("Failed to close the client")
-	} else {
-		Logf("Closed the client")
-	}
-
-	// instance.DeleteInstance()
-})
-
 var _ = Describe("GCE PD CSI Driver", func() {
 
-	BeforeEach(func() {
-		err := client.AssertCSIConnection()
-		Expect(err).To(BeNil(), "Failed to assert csi client connection: %v", err)
-	})
-
 	It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() {
+		// Create new driver and client
+		// TODO: Should probably actual have some object that includes both client and instance so we can relate the two??
+		Expect(testInstances).NotTo(BeEmpty())
+		testContext, err := utils.SetupNewDriverAndClient(testInstances[0])
+		Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
+		p, z, _ := testContext.Instance.GetIdentity()
+		client := testContext.Client
+		instance := testContext.Instance
+
 		// Create Disk
 		volName := testNamePrefix + string(uuid.NewUUID())
-		volId, err := client.CreateVolume(volName)
+		volId, err := client.CreateVolume(volName, nil)
 		Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
 
 		// TODO: Validate Disk Created
-		/*cloudDisk, err := gceCloud.GetDiskOrError(context.Background(), gceCloud.GetZone(), volName)
+		cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
 		Expect(err).To(BeNil(), "Could not get disk from cloud directly")
 		Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
 		Expect(cloudDisk.Status).To(Equal(readyState))
 		Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
-		Expect(cloudDisk.Name).To(Equal(volName))*/
+		Expect(cloudDisk.Name).To(Equal(volName))
 
 		defer func() {
 			// Delete Disk
 			client.DeleteVolume(volId)
 			Expect(err).To(BeNil(), "DeleteVolume failed")
 
 			// TODO: Validate Disk Deleted
-			/*_, err = gceCloud.GetDiskOrError(context.Background(), gceCloud.GetZone(), volName)
-			serverError, ok := status.FromError(err)
-			Expect(ok).To(BeTrue())
-			Expect(serverError.Code()).To(Equal(codes.NotFound))*/
+			_, err = computeService.Disks.Get(p, z, volName).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
 		}()
 
 		// Attach Disk
-		err = client.ControllerPublishVolume(volId, nodeID)
+		err = client.ControllerPublishVolume(volId, instance.GetName())
 		Expect(err).To(BeNil(), "ControllerPublishVolume failed with error")
 
 		defer func() {
 			// Detach Disk
-			err := client.ControllerUnpublishVolume(volId, nodeID)
+			err := client.ControllerUnpublishVolume(volId, instance.GetName())
 			Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
 		}()
 
@@ -166,6 +131,37 @@ var _ = Describe("GCE PD CSI Driver", func() {
 
 	})
 
+	It("Should create disks in correct zones when toplogy is specified", func() {
+		///
+		Expect(testInstances).NotTo(BeEmpty())
+		testContext, err := utils.SetupNewDriverAndClient(testInstances[0])
+		Expect(err).To(BeNil(), "Failed to set up new driver and client")
+		p, _, _ := testContext.Instance.GetIdentity()
+
+		zones := []string{"us-central1-c", "us-central1-b", "us-central1-a"}
+
+		for _, zone := range zones {
+			volName := testNamePrefix + string(uuid.NewUUID())
+			topReq := &csi.TopologyRequirement{
+				Requisite: []*csi.Topology{
+					{
+						Segments: map[string]string{"zone": zone},
+					},
+				},
+			}
+			volID, err := testContext.Client.CreateVolume(volName, topReq)
+			Expect(err).To(BeNil(), "Failed to create volume")
+			defer func() {
+				err = testContext.Client.DeleteVolume(volID)
+				Expect(err).To(BeNil(), "Failed to delete volume")
+			}()
+
+			_, err = computeService.Disks.Get(p, zone, volName).Do()
+			Expect(err).To(BeNil(), "Could not find disk in correct zone")
+		}
+
+	})
+
 	// Test volume already exists
 
 	// Test volume with op pending