Skip to content

Commit bc201a2

Browse files
authored
Merge pull request #80 from davidz627/feature/e2eInterface
Refactored remote test framework for easy usage. Added teardown methods
2 parents da1b1c0 + e6dc8b0 commit bc201a2

10 files changed

+206
-69
lines changed

test/e2e/tests/multi_zone_e2e_test.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
. "github.com/onsi/gomega"
2020
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
2121
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
22+
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
2223
)
2324

2425
var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
@@ -30,8 +31,12 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
3031

3132
It("Should get reasonable topology from nodes with NodeGetInfo", func() {
3233
for _, instance := range testInstances {
33-
testContext, err := testutils.SetupNewDriverAndClient(instance)
34+
testContext, err := testutils.GCEClientAndDriverSetup(instance)
3435
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
36+
defer func() {
37+
err := remote.TeardownDriverAndClient(testContext)
38+
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
39+
}()
3540

3641
resp, err := testContext.Client.NodeGetInfo()
3742
Expect(err).To(BeNil())

test/e2e/tests/setup_e2e_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ import (
2424
. "github.com/onsi/ginkgo"
2525
. "github.com/onsi/gomega"
2626
compute "google.golang.org/api/compute/v1"
27-
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
2827
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
28+
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
2929
)
3030

3131
var (
@@ -63,7 +63,7 @@ var _ = BeforeSuite(func() {
6363
Expect(*project).ToNot(BeEmpty(), "Project should not be empty")
6464
Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty")
6565

66-
i, err := testutils.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
66+
i, err := remote.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
6767
Expect(err).To(BeNil())
6868

6969
testInstances = append(testInstances, i)

test/e2e/tests/single_zone_e2e_test.go

+13-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
2424
gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
2525
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
26+
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
2627

2728
csi "github.com/container-storage-interface/spec/lib/go/csi/v0"
2829
. "github.com/onsi/ginkgo"
@@ -44,8 +45,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
4445
// Create new driver and client
4546
// TODO: Should probably actual have some object that includes both client and instance so we can relate the two??
4647
Expect(testInstances).NotTo(BeEmpty())
47-
testContext, err := testutils.SetupNewDriverAndClient(testInstances[0])
48+
testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
4849
Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
50+
defer func() {
51+
err := remote.TeardownDriverAndClient(testContext)
52+
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
53+
}()
54+
4955
p, z, _ := testContext.Instance.GetIdentity()
5056
client := testContext.Client
5157
instance := testContext.Instance
@@ -135,8 +141,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
135141
It("Should create disks in correct zones when topology is specified", func() {
136142
///
137143
Expect(testInstances).NotTo(BeEmpty())
138-
testContext, err := testutils.SetupNewDriverAndClient(testInstances[0])
144+
testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
139145
Expect(err).To(BeNil(), "Failed to set up new driver and client")
146+
defer func() {
147+
err := remote.TeardownDriverAndClient(testContext)
148+
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
149+
}()
150+
140151
p, _, _ := testContext.Instance.GetIdentity()
141152

142153
zones := []string{"us-central1-c", "us-central1-b", "us-central1-a"}

test/e2e/utils/utils.go

+10-53
Original file line numberDiff line numberDiff line change
@@ -25,81 +25,38 @@ import (
2525
"github.com/golang/glog"
2626
"golang.org/x/oauth2/google"
2727
cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
28-
compute "google.golang.org/api/compute/v1"
2928
boskosclient "k8s.io/test-infra/boskos/client"
30-
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
29+
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
3130
)
3231

3332
var (
3433
boskos = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos")
3534
)
3635

37-
const (
38-
archiveName = "e2e_gce_pd_test.tar.gz"
39-
)
40-
41-
type TestContext struct {
42-
Instance *remote.InstanceInfo
43-
Client *CsiClient
44-
}
45-
46-
func SetupInstance(instanceProject, instanceZone, instanceName, instanceServiceAccount string, cs *compute.Service) (*remote.InstanceInfo, error) {
47-
// Create the instance in the requisite zone
48-
instance, err := remote.CreateInstanceInfo(instanceProject, instanceZone, instanceName, cs)
49-
if err != nil {
50-
return nil, err
51-
}
52-
53-
err = instance.CreateOrGetInstance(instanceServiceAccount)
54-
if err != nil {
55-
return nil, err
56-
}
57-
return instance, nil
58-
}
59-
60-
// TODO: Need a function to clean up this driver from the instance
61-
func SetupNewDriverAndClient(instance *remote.InstanceInfo) (*TestContext, error) {
36+
func GCEClientAndDriverSetup(instance *remote.InstanceInfo) (*remote.TestContext, error) {
6237
port := fmt.Sprintf("%v", 1024+rand.Intn(10000))
6338
goPath, ok := os.LookupEnv("GOPATH")
6439
if !ok {
6540
return nil, fmt.Errorf("Could not find environment variable GOPATH")
6641
}
6742
pkgPath := path.Join(goPath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/")
6843
binPath := path.Join(pkgPath, "bin/gce-pd-csi-driver")
69-
archivePath, err := remote.CreateDriverArchive(archiveName, pkgPath, binPath)
70-
if err != nil {
71-
return nil, err
72-
}
73-
defer func() {
74-
err = os.Remove(archivePath)
75-
if err != nil {
76-
glog.Warningf("Failed to remove archive file %s: %v", archivePath, err)
77-
}
78-
}()
7944

80-
// Upload archive to instance and run binaries
8145
endpoint := fmt.Sprintf("tcp://localhost:%s", port)
46+
8247
workspace := remote.NewWorkspaceDir("gce-pd-e2e-")
8348
driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver --endpoint=%s --nodeid=%s > %s/prog.out 2> %s/prog.err < /dev/null &'",
8449
workspace, endpoint, instance.GetName(), workspace, workspace)
85-
err = instance.UploadAndRun(archivePath, workspace, driverRunCmd)
86-
if err != nil {
87-
return nil, err
88-
}
8950

90-
// Create an SSH tunnel from port to port
91-
res, err := instance.CreateSSHTunnel(port, port)
92-
if err != nil {
93-
return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", res, err)
94-
}
95-
96-
client := CreateCSIClient(fmt.Sprintf("localhost:%s", port))
97-
err = client.AssertCSIConnection()
98-
if err != nil {
99-
return nil, fmt.Errorf("asserting csi connection failed with: %v", err)
51+
config := &remote.ClientConfig{
52+
PkgPath: pkgPath,
53+
BinPath: binPath,
54+
WorkspaceDir: workspace,
55+
RunDriverCmd: driverRunCmd,
56+
Port: port,
10057
}
10158

102-
return &TestContext{Instance: instance, Client: client}, nil
59+
return remote.SetupNewDriverAndClient(instance, config)
10360
}
10461

10562
func SetupProwConfig() (project, serviceAccount string) {

test/binremote/archiver.go renamed to test/remote/archiver.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package binremote
17+
package remote
1818

1919
import (
2020
"fmt"

test/e2e/utils/client_wrappers.go renamed to test/remote/client-wrappers.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
1212
limitations under the License.
1313
*/
1414

15-
package utils
15+
package remote
1616

1717
import (
1818
"context"

test/binremote/instance.go renamed to test/remote/instance.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package binremote
17+
package remote
1818

1919
import (
2020
"context"

test/binremote/runner.go renamed to test/remote/runner.go

+27-7
Original file line numberDiff line numberDiff line change
@@ -14,31 +14,33 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package binremote
17+
package remote
1818

1919
import (
2020
"fmt"
2121
"path"
2222
"path/filepath"
23+
"strconv"
24+
"strings"
2325

2426
"github.com/golang/glog"
2527
)
2628

27-
func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) error {
29+
func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) (int, error) {
2830

2931
// Create the temp staging directory
3032
glog.V(4).Infof("Staging test binaries on %q", i.name)
3133

3234
// Do not sudo here, so that we can use scp to copy test archive to the directdory.
3335
if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil {
3436
// Exit failure with the error
35-
return fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output)
37+
return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output)
3638
}
3739

3840
// Copy the archive to the staging directory
3941
if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), remoteWorkspace)); err != nil {
4042
// Exit failure with the error
41-
return fmt.Errorf("failed to copy test archive: %v, output: %q", err, output)
43+
return -1, fmt.Errorf("failed to copy test archive: %v, output: %q", err, output)
4244
}
4345

4446
// Extract the archive
@@ -52,22 +54,40 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s
5254
// we want the extracted files to be owned by the current user.
5355
if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil {
5456
// Exit failure with the error
55-
return fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
57+
return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
5658
}
5759

5860
glog.V(4).Infof("Starting driver on %q", i.name)
5961
// When the process is killed the driver should close the TCP endpoint, then we want to download the logs
6062
output, err := i.SSH(driverRunCmd)
63+
if err != nil {
64+
// Exit failure with the error
65+
return -1, fmt.Errorf("failed start driver, got output: %v, error: %v", output, err)
66+
}
6167

68+
// Get the driver PID
69+
// ps -aux | grep /tmp/gce-pd-e2e-0180801T114407/gce-pd-csi-driver | awk '{print $2}'
70+
driverPIDCmd := getSSHCommand(" | ",
71+
"ps -aux",
72+
fmt.Sprintf("grep %s", remoteWorkspace),
73+
"grep -v grep",
74+
// All ye who try to deal with escaped/non-escaped quotes with exec beware.
75+
//`awk "{print \$2}"`,
76+
)
77+
driverPIDString, err := i.SSHNoSudo("sh", "-c", driverPIDCmd)
6278
if err != nil {
6379
// Exit failure with the error
64-
return fmt.Errorf("failed start GCE PD driver, got output: %v, error: %v", output, err)
80+
return -1, fmt.Errorf("failed to get PID of driver, got output: %v, error: %v", output, err)
6581
}
6682

83+
driverPID, err := strconv.Atoi(strings.Fields(driverPIDString)[1])
84+
if err != nil {
85+
return -1, fmt.Errorf("failed to convert driver PID from string %s to int: %v", driverPIDString, err)
86+
}
6787
// TODO: return the PID so that we can kill the driver later
6888
// Actually just do a pkill -f my_pattern
6989

70-
return nil
90+
return driverPID, nil
7191
}
7292

7393
func NewWorkspaceDir(workspaceDirPrefix string) string {

0 commit comments

Comments
 (0)