Skip to content

Test contexts are now initialized once on startup in parallel instead of per-test, Chose a better boot disk image. #104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 28, 2018
Merged

Test contexts are now initialized once on startup in parallel instead of per-test, Chose a better boot disk image. #104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6f703a
Test contexts are now initialized once on startup instead of per-test
davidz627 Aug 27, 2018
4198d90
Enable test setup to run in parallel
davidz627 Aug 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 21 additions & 35 deletions test/e2e/tests/multi_zone_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,11 @@ import (

var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
BeforeEach(func() {
Expect(len(testInstances)).To(BeNumerically(">", 1))
Expect(len(testContexts)).To(BeNumerically(">", 1))
})

It("Should get reasonable topology from nodes with NodeGetInfo", func() {
for _, instance := range testInstances {
testContext, err := testutils.GCEClientAndDriverSetup(instance)
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
defer func() {
err := remote.TeardownDriverAndClient(testContext)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
}()

for _, testContext := range testContexts {
resp, err := testContext.Client.NodeGetInfo()
Expect(err).To(BeNil())

Expand All @@ -65,24 +58,17 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
It("Should successfully run through entire lifecycle of an RePD volume on instances in 2 zones", func() {
// Create new driver and client

Expect(testInstances).NotTo(BeEmpty())
Expect(testContexts).NotTo(BeEmpty())

zoneToContext := map[string]*remote.TestContext{}
zones := []string{}

for _, i := range testInstances {
_, z, _ := i.GetIdentity()
for _, tc := range testContexts {
_, z, _ := tc.Instance.GetIdentity()
// Zone hasn't been seen before
if _, ok := zoneToContext[z]; !ok {
c, err := testutils.GCEClientAndDriverSetup(i)
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
zoneToContext[z] = c
zoneToContext[z] = tc
zones = append(zones, z)

defer func() {
err := remote.TeardownDriverAndClient(c)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
}()
}
if len(zoneToContext) == 2 {
break
Expand All @@ -102,7 +88,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {

// Create Disk
volName := testNamePrefix + string(uuid.NewUUID())
volId, err := controllerClient.CreateVolume(volName, map[string]string{
volID, err := controllerClient.CreateVolume(volName, map[string]string{
common.ParameterKeyReplicationType: "regional-pd",
}, defaultSizeGb, &csi.TopologyRequirement{
Requisite: []*csi.Topology{
Expand Down Expand Up @@ -133,7 +119,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {

defer func() {
// Delete Disk
controllerClient.DeleteVolume(volId)
controllerClient.DeleteVolume(volID)
Expect(err).To(BeNil(), "DeleteVolume failed")

// Validate Disk Deleted
Expand All @@ -148,45 +134,45 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
if i >= 1 {
readOnly = true
}
testAttachWriteReadDetach(volId, volName, testContext.Instance, testContext.Client, readOnly)
testAttachWriteReadDetach(volID, volName, testContext.Instance, testContext.Client, readOnly)
i = i + 1
}

})

})

func testAttachWriteReadDetach(volId string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
var err error

Logf("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volId, instance.GetNodeID(), readOnly)
Logf("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
// Attach Disk
err = client.ControllerPublishVolume(volId, instance.GetNodeID())
Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volId, instance.GetNodeID())
err = client.ControllerPublishVolume(volID, instance.GetNodeID())
Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())

defer func() {

// Detach Disk
err = client.ControllerUnpublishVolume(volId, instance.GetNodeID())
err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
}()

// Stage Disk
stageDir := filepath.Join("/tmp/", volName, "stage")
client.NodeStageVolume(volId, stageDir)
client.NodeStageVolume(volID, stageDir)
Expect(err).To(BeNil(), "NodeStageVolume failed with error")

defer func() {
// Unstage Disk
err = client.NodeUnstageVolume(volId, stageDir)
err = client.NodeUnstageVolume(volID, stageDir)
Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
Expect(err).To(BeNil(), "Failed to remove temp directory")
}()

// Mount Disk
publishDir := filepath.Join("/tmp/", volName, "mount")
err = client.NodePublishVolume(volId, stageDir, publishDir)
err = client.NodePublishVolume(volID, stageDir, publishDir)
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
Expect(err).To(BeNil(), "Chmod failed with error")
Expand All @@ -199,12 +185,12 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In
}

// Unmount Disk
err = client.NodeUnpublishVolume(volId, publishDir)
err = client.NodeUnpublishVolume(volID, publishDir)
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")

// Mount disk somewhere else
secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
err = client.NodePublishVolume(volId, stageDir, secondPublishDir)
err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
Expect(err).To(BeNil(), "Chmod failed with error")
Expand All @@ -216,8 +202,8 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In
Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))

// Unmount Disk
err = client.NodeUnpublishVolume(volId, secondPublishDir)
err = client.NodeUnpublishVolume(volID, secondPublishDir)
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")

Logf("Completed testAttachWriteReadDetach with volume %v node %v\n", volId, instance.GetNodeID())
Logf("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
}
49 changes: 33 additions & 16 deletions test/e2e/tests/setup_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ var (
runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run")

testInstances = []*remote.InstanceInfo{}
testContexts = []*remote.TestContext{}
computeService *compute.Service
betaComputeService *computebeta.Service
)
Expand All @@ -48,6 +48,9 @@ func TestE2E(t *testing.T) {

var _ = BeforeSuite(func() {
var err error
tcc := make(chan *remote.TestContext)
defer close(tcc)

zones := []string{"us-central1-c", "us-central1-b"}

rand.Seed(time.Now().UnixNano())
Expand All @@ -68,27 +71,41 @@ var _ = BeforeSuite(func() {
Logf("Running in project %v with service account %v\n\n", *project, *serviceAccount)

for _, zone := range zones {
nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)

i, err := remote.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
Expect(err).To(BeNil())

testInstances = append(testInstances, i)
go func(curZone string) {
defer GinkgoRecover()
nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", curZone)
Logf("Setting up node %s\n", nodeID)

i, err := remote.SetupInstance(*project, curZone, nodeID, *serviceAccount, computeService)
Expect(err).To(BeNil())

// Create new driver and client
testContext, err := testutils.GCEClientAndDriverSetup(i)
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
tcc <- testContext
}(zone)
}

for i := 0; i < len(zones); i++ {
tc := <-tcc
Logf("Test Context for node %s set up\n", tc.Instance.GetName())
testContexts = append(testContexts, tc)
}
})

var _ = AfterSuite(func() {
/*
err := node.client.CloseConn()
if err != nil {
Logf("Failed to close the client")
} else {
Logf("Closed the client")
*/
for _, i := range testInstances {

for _, tc := range testContexts {
err := remote.TeardownDriverAndClient(tc)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
if *deleteInstances {
i.DeleteInstance()
tc.Instance.DeleteInstance()
}
}
})

func getRandomTestContext() *remote.TestContext {
Expect(testContexts).ToNot(BeEmpty())
rn := rand.Intn(len(testContexts))
return testContexts[rn]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any issue if multiple tests run in parallel and choose the same context?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't believe so, since the driver can handle multiple incoming requests and all volume's should be unique due to UUID's. If there is a parallelism issue here it would be with the driver, not the tests and that is something we'd want to catch if it existed.

}
55 changes: 14 additions & 41 deletions test/e2e/tests/single_zone_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"

csi "github.com/container-storage-interface/spec/lib/go/csi/v0"
. "github.com/onsi/ginkgo"
Expand All @@ -41,22 +39,15 @@ const (
var _ = Describe("GCE PD CSI Driver", func() {

It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() {
// Create new driver and client
Expect(testInstances).NotTo(BeEmpty())
testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
defer func() {
err := remote.TeardownDriverAndClient(testContext)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
}()
testContext := getRandomTestContext()

p, z, _ := testContext.Instance.GetIdentity()
client := testContext.Client
instance := testContext.Instance

// Create Disk
volName := testNamePrefix + string(uuid.NewUUID())
volId, err := client.CreateVolume(volName, nil, defaultSizeGb,
volID, err := client.CreateVolume(volName, nil, defaultSizeGb,
&csi.TopologyRequirement{
Requisite: []*csi.Topology{
{
Expand All @@ -76,7 +67,7 @@ var _ = Describe("GCE PD CSI Driver", func() {

defer func() {
// Delete Disk
client.DeleteVolume(volId)
client.DeleteVolume(volID)
Expect(err).To(BeNil(), "DeleteVolume failed")

// Validate Disk Deleted
Expand All @@ -85,19 +76,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
}()

// Attach Disk
testAttachWriteReadDetach(volId, volName, instance, client, false /* readOnly */)
testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)

})

It("Should create disks in correct zones when topology is specified", func() {
///
Expect(testInstances).NotTo(BeEmpty())
testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
Expect(err).To(BeNil(), "Failed to set up new driver and client")
defer func() {
err := remote.TeardownDriverAndClient(testContext)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
}()
Expect(testContexts).ToNot(BeEmpty())
testContext := getRandomTestContext()

p, _, _ := testContext.Instance.GetIdentity()

Expand Down Expand Up @@ -126,14 +111,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
})

It("Should successfully create RePD in two zones in the drivers region when none are specified", func() {
// Create new driver and client
Expect(testInstances).NotTo(BeEmpty())
testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
Expect(err).To(BeNil(), "Failed to set up new driver and client")
defer func() {
err := remote.TeardownDriverAndClient(testContext)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
}()
Expect(testContexts).ToNot(BeEmpty())
testContext := getRandomTestContext()

controllerInstance := testContext.Instance
controllerClient := testContext.Client
Expand All @@ -145,7 +124,7 @@ var _ = Describe("GCE PD CSI Driver", func() {

// Create Disk
volName := testNamePrefix + string(uuid.NewUUID())
volId, err := controllerClient.CreateVolume(volName, map[string]string{
volID, err := controllerClient.CreateVolume(volName, map[string]string{
common.ParameterKeyReplicationType: "regional-pd",
}, defaultSizeGb, nil)
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
Expand All @@ -167,7 +146,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
}
defer func() {
// Delete Disk
controllerClient.DeleteVolume(volId)
controllerClient.DeleteVolume(volID)
Expect(err).To(BeNil(), "DeleteVolume failed")

// Validate Disk Deleted
Expand All @@ -177,21 +156,15 @@ var _ = Describe("GCE PD CSI Driver", func() {
})

It("Should create and delete disk with default zone", func() {
// Create new driver and client
Expect(testInstances).NotTo(BeEmpty())
testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
defer func() {
err := remote.TeardownDriverAndClient(testContext)
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
}()
Expect(testContexts).ToNot(BeEmpty())
testContext := getRandomTestContext()

p, z, _ := testContext.Instance.GetIdentity()
client := testContext.Client

// Create Disk
volName := testNamePrefix + string(uuid.NewUUID())
volId, err := client.CreateVolume(volName, nil, defaultSizeGb, nil)
volID, err := client.CreateVolume(volName, nil, defaultSizeGb, nil)
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)

// Validate Disk Created
Expand All @@ -204,7 +177,7 @@ var _ = Describe("GCE PD CSI Driver", func() {

defer func() {
// Delete Disk
client.DeleteVolume(volId)
client.DeleteVolume(volID)
Expect(err).To(BeNil(), "DeleteVolume failed")

// Validate Disk Deleted
Expand Down
3 changes: 1 addition & 2 deletions test/remote/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ func (i *InstanceInfo) CreateOrGetInstance(serviceAccount string) error {
return fmt.Errorf("Failed to create firewall rule: %v", err)
}

// TODO(#97): Pick a better boot disk image
imageURL := "projects/ml-images/global/images/family/tf-1-9"
imageURL := "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-1804-lts"
inst := &compute.Instance{
Name: i.name,
MachineType: machineType(i.zone, ""),
Expand Down
6 changes: 2 additions & 4 deletions test/remote/setup-teardown.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,7 @@ import (

"github.com/golang/glog"
compute "google.golang.org/api/compute/v1"
)

const (
archiveName = "e2e_driver_binaries.tar.gz"
"k8s.io/apimachinery/pkg/util/uuid"
)

// TestContext holds the CSI Client handle to a remotely connected Driver
Expand Down Expand Up @@ -75,6 +72,7 @@ func SetupInstance(instanceProject, instanceZone, instanceName, instanceServiceA
// a CSI client to it through SHH tunnelling. It returns a TestContext with both a handle to the instance
// that the driver is on and the CSI Client object to make CSI calls to the remote driver.
func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*TestContext, error) {
archiveName := fmt.Sprintf("e2e_driver_binaries_%s.tar.gz", uuid.NewUUID())
archivePath, err := CreateDriverArchive(archiveName, config.PkgPath, config.BinPath)
if err != nil {
return nil, err
Expand Down