diff --git a/Makefile b/Makefile index 269d34490..cd3660a8c 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,6 @@ all: gce-pd-driver gce-pd-driver: mkdir -p bin go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/gce-pd-csi-driver ./cmd/ - go test -c sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e -o bin/e2e.test build-container: docker build --build-arg TAG=$(STAGINGVERSION) -t $(STAGINGIMAGE):$(STAGINGVERSION) . diff --git a/pkg/gce-pd-csi-driver/server.go b/pkg/gce-pd-csi-driver/server.go index 47492ebcc..232f68277 100644 --- a/pkg/gce-pd-csi-driver/server.go +++ b/pkg/gce-pd-csi-driver/server.go @@ -70,6 +70,9 @@ func (s *nonBlockingGRPCServer) ForceStop() { } func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(logGRPC), + } u, err := url.Parse(endpoint) @@ -83,6 +86,8 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { glog.Fatalf("Failed to remove %s, error: %s", addr, err.Error()) } + } else if u.Scheme == "tcp" { + addr = u.Host } else { glog.Fatalf("%v endpoint scheme not supported", u.Scheme) } @@ -93,9 +98,6 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c glog.Fatalf("Failed to listen: %v", err) } - opts := []grpc.ServerOption{ - grpc.UnaryInterceptor(logGRPC), - } server := grpc.NewServer(opts...) s.server = server @@ -111,6 +113,8 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c glog.Infof("Listening for connections on address: %#v", listener.Addr()) - server.Serve(listener) + if err := server.Serve(listener); err != nil { + glog.Fatalf("Failed to serve: %v", err) + } } diff --git a/test/binremote/archiver.go b/test/binremote/archiver.go new file mode 100644 index 000000000..60733e4ce --- /dev/null +++ b/test/binremote/archiver.go @@ -0,0 +1,73 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binremote + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + "github.com/golang/glog" +) + +func CreateDriverArchive(archiveName, pkgPath, binPath string) (string, error) { + glog.V(2).Infof("Building archive...") + tarDir, err := ioutil.TempDir("", "driver-temp-archive") + if err != nil { + return "", fmt.Errorf("failed to create temporary directory %v", err) + } + defer os.RemoveAll(tarDir) + + // Call the suite function to setup the test package. + err = setupBinaries(tarDir, pkgPath, binPath) + if err != nil { + return "", fmt.Errorf("failed to setup test package %q: %v", tarDir, err) + } + + // Build the tar + out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tarDir, ".").CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err, out) + } + + dir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get working directory %v", err) + } + return filepath.Join(dir, archiveName), nil +} + +func setupBinaries(tarDir, pkgPath, binPath string) error { + glog.V(4).Infof("Making binaries and copying to temp dir...") + out, err := exec.Command("make", "-C", pkgPath).CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to make at %s: %v: %v", pkgPath, string(out), err) + } + + // Copy binaries + if _, err := os.Stat(binPath); err != nil { + return fmt.Errorf("failed to locate test binary %s: %v", binPath, err) + } + out, err = exec.Command("cp", binPath, tarDir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy %q: %v Output: %q", binPath, err, out) + } + + return nil +} diff --git a/test/binremote/instance.go b/test/binremote/instance.go new file mode 100644 index 000000000..679e4acb6 --- /dev/null +++ b/test/binremote/instance.go @@ -0,0 +1,308 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binremote + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "time" + + "github.com/golang/glog" + "golang.org/x/oauth2/google" + compute "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + defaultMachine = "n1-standard-1" + defaultFirewallRule = "default-allow-ssh" + + // timestampFormat is the timestamp format used in the e2e directory name. + timestampFormat = "20060102T150405" +) + +type InstanceInfo struct { + project string + zone string + name string + + externalIP string + + computeService *compute.Service +} + +func CreateInstanceInfo(project, zone, name string) (*InstanceInfo, error) { + cs, err := getComputeClient() + if err != nil { + return nil, err + } + return &InstanceInfo{ + project: project, + zone: zone, + name: name, + + computeService: cs, + }, nil +} + +// Provision a gce instance using image +func (i *InstanceInfo) CreateInstance(serviceAccount string) error { + var err error + var instance *compute.Instance + glog.V(4).Infof("Creating instance: %v", i.name) + + myuuid := string(uuid.NewUUID()) + + err = i.createDefaultFirewallRule() + if err != nil { + return fmt.Errorf("Failed to create firewall rule: %v", err) + } + + // TODO: Pick a better boot disk image + imageURL := "projects/ml-images/global/images/family/tf-1-9" + inst := &compute.Instance{ + Name: i.name, + MachineType: machineType(i.zone, ""), + NetworkInterfaces: []*compute.NetworkInterface{ + { + AccessConfigs: []*compute.AccessConfig{ + { + Type: "ONE_TO_ONE_NAT", + Name: "External NAT", + }, + }}, + }, + Disks: []*compute.AttachedDisk{ + { + AutoDelete: true, + Boot: true, + Type: "PERSISTENT", + InitializeParams: &compute.AttachedDiskInitializeParams{ + DiskName: "my-root-pd-" + myuuid, + SourceImage: imageURL, + }, + }, + }, + } + + saObj := &compute.ServiceAccount{ + Email: serviceAccount, + Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"}, + } + inst.ServiceAccounts = []*compute.ServiceAccount{saObj} + + if pubkey, ok := os.LookupEnv("JENKINS_GCE_SSH_PUBLIC_KEY_FILE"); ok { + glog.V(4).Infof("JENKINS_GCE_SSH_PUBLIC_KEY_FILE set to %v, adding public key to Instance", pubkey) + meta, err := generateMetadataWithPublicKey(pubkey) + if err != nil { + return err + } + inst.Metadata = meta + } + + if _, err := i.computeService.Instances.Get(i.project, i.zone, inst.Name).Do(); err != nil { + op, err := i.computeService.Instances.Insert(i.project, i.zone, inst).Do() + glog.V(4).Infof("Inserted instance %v in project %v, i.zone %v", inst.Name, i.project, i.zone) + if err != nil { + ret := fmt.Sprintf("could not create instance %s: API error: %v", i.name, err) + if op != nil { + ret = fmt.Sprintf("%s: %v", ret, op.Error) + } + return fmt.Errorf(ret) + } else if op.Error != nil { + return fmt.Errorf("could not create instance %s: %+v", i.name, op.Error) + } + } else { + glog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", inst.Name) + } + + then := time.Now() + err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { + glog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(then)) + + instance, err = i.computeService.Instances.Get(i.project, i.zone, i.name).Do() + if err != nil { + glog.Errorf("Failed to get instance %v: %v", i.name, err) + return false, nil + } + + if strings.ToUpper(instance.Status) != "RUNNING" { + glog.Warningf("instance %s not in state RUNNING, was %s", i.name, instance.Status) + return false, nil + } + + externalIP := getexternalIP(instance) + if len(externalIP) > 0 { + i.externalIP = externalIP + } + + if sshOut, err := i.SSHCheckAlive(); err != nil { + err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.name, err) + glog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut) + return false, nil + } + glog.Infof("Instance %v in state RUNNING and vailable by SSH", i.name) + return true, nil + }) + + // If instance didn't reach running state in time, return with error now. + if err != nil { + return err + } + + // Instance reached running state in time, make sure that cloud-init is complete + glog.V(2).Infof("Instance %v has been created successfully", i.name) + return nil +} + +func (i *InstanceInfo) DeleteInstance() { + glog.V(4).Infof("Deleting instance %q", i.name) + _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do() + if err != nil { + if isGCEError(err, "notFound") { + return + } + glog.Errorf("Error deleting instance %q: %v", i.name, err) + } +} + +func getexternalIP(instance *compute.Instance) string { + for i := range instance.NetworkInterfaces { + ni := instance.NetworkInterfaces[i] + for j := range ni.AccessConfigs { + ac := ni.AccessConfigs[j] + if len(ac.NatIP) > 0 { + return ac.NatIP + } + } + } + return "" +} + +func getTimestamp() string { + return fmt.Sprintf(time.Now().Format(timestampFormat)) +} + +func machineType(zone, machine string) string { + if machine == "" { + machine = defaultMachine + } + return fmt.Sprintf("zones/%s/machineTypes/%s", zone, machine) +} + +// Create default SSH filewall rule if it does not exist +func (i *InstanceInfo) createDefaultFirewallRule() error { + var err error + glog.V(4).Infof("Creating default firewall rule %s...", defaultFirewallRule) + + if _, err = i.computeService.Firewalls.Get(i.project, defaultFirewallRule).Do(); err != nil { + glog.Infof("Default firewall rule %v does not exist, creating", defaultFirewallRule) + f := &compute.Firewall{ + Name: defaultFirewallRule, + Allowed: []*compute.FirewallAllowed{ + { + IPProtocol: "tcp", + Ports: []string{"22"}, + }, + }, + } + _, err = i.computeService.Firewalls.Insert(i.project, f).Do() + if err != nil { + return fmt.Errorf("Failed to insert required default SSH firewall Rule %v: %v", defaultFirewallRule, err) + } + } else { + glog.Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) + } + return nil +} + +func getComputeClient() (*compute.Service, error) { + const retries = 10 + const backoff = time.Second * 6 + + glog.V(4).Infof("Getting compute client...") + + // Setup the gce client for provisioning instances + // Getting credentials on gce jenkins is flaky, so try a couple times + var err error + var cs *compute.Service + for i := 0; i < retries; i++ { + if i > 0 { + time.Sleep(backoff) + } + + var client *http.Client + client, err = google.DefaultClient(context.TODO(), compute.ComputeScope) + if err != nil { + continue + } + + cs, err = compute.New(client) + if err != nil { + continue + } + return cs, nil + } + return nil, err +} + +func generateMetadataWithPublicKey(pubKeyFile string) (*compute.Metadata, error) { + publicKeyByte, err := ioutil.ReadFile(pubKeyFile) + if err != nil { + return nil, err + } + + publicKey := string(publicKeyByte) + + // Take username and prepend it to the public key + tokens := strings.Split(publicKey, " ") + if len(tokens) != 3 { + return nil, fmt.Errorf("Public key not comprised of 3 parts, instead was: %v", publicKey) + } + publicKey = strings.TrimSpace(tokens[2]) + ":" + publicKey + newMeta := &compute.Metadata{ + Items: []*compute.MetadataItems{ + { + Key: "ssh-keys", + Value: &publicKey, + }, + }, + } + return newMeta, nil +} + +// isGCEError returns true if given error is a googleapi.Error with given +// reason (e.g. "resourceInUseByAnotherResource") +func isGCEError(err error, reason string) bool { + apiErr, ok := err.(*googleapi.Error) + if !ok { + return false + } + + for _, e := range apiErr.Errors { + if e.Reason == reason { + return true + } + } + return false +} diff --git a/test/binremote/runner.go b/test/binremote/runner.go new file mode 100644 index 000000000..5ca5e7bec --- /dev/null +++ b/test/binremote/runner.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binremote + +import ( + "fmt" + "path" + "path/filepath" + + "github.com/golang/glog" +) + +func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) error { + + // Create the temp staging directory + glog.V(4).Infof("Staging test binaries on %q", i.name) + + // Do not sudo here, so that we can use scp to copy test archive to the directdory. + if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil { + // Exit failure with the error + return fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output) + } + + // Copy the archive to the staging directory + if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), remoteWorkspace)); err != nil { + // Exit failure with the error + return fmt.Errorf("failed to copy test archive: %v, output: %q", err, output) + } + + // Extract the archive + archiveName := path.Base(archivePath) + cmd := getSSHCommand(" && ", + fmt.Sprintf("cd %s", remoteWorkspace), + fmt.Sprintf("tar -xzvf ./%s", archiveName), + ) + glog.V(4).Infof("Extracting tar on %q", i.name) + // Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but + // we want the extracted files to be owned by the current user. + if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil { + // Exit failure with the error + return fmt.Errorf("failed to extract test archive: %v, output: %q", err, output) + } + + glog.V(4).Infof("Starting driver on %q", i.name) + // When the process is killed the driver should close the TCP endpoint, then we want to download the logs + output, err := i.SSH(driverRunCmd) + + if err != nil { + // Exit failure with the error + return fmt.Errorf("failed start GCE PD driver, got output: %v, error: %v", output, err) + } + + // TODO: return the PID so that we can kill the driver later + // Actually just do a pkill -f my_pattern + + return nil +} + +func NewWorkspaceDir(workspaceDirPrefix string) string { + return filepath.Join("/tmp", workspaceDirPrefix+getTimestamp()) +} diff --git a/test/binremote/ssh.go b/test/binremote/ssh.go new file mode 100644 index 000000000..5b39b2df2 --- /dev/null +++ b/test/binremote/ssh.go @@ -0,0 +1,112 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binremote + +import ( + "fmt" + "os" + "os/exec" + "os/user" + "strings" + + "github.com/golang/glog" +) + +var ( + sshOption = "-o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o LogLevel=ERROR" + sshDefaultKey string +) + +func init() { + usr, err := user.Current() + if err != nil { + glog.Fatal(err) + } + sshDefaultKey = fmt.Sprintf("%s/.ssh/google_compute_engine", usr.HomeDir) + +} + +// GetHostnameOrIP converts hostname into ip and apply user if necessary. +func (i *InstanceInfo) GetSSHTarget() string { + var target string + + if _, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { + target = fmt.Sprintf("prow@%s", i.externalIP) + } else { + target = fmt.Sprintf("%s", i.externalIP) + } + return target +} + +// getSSHCommand handles proper quoting so that multiple commands are executed in the same shell over ssh +func getSSHCommand(sep string, args ...string) string { + return fmt.Sprintf("'%s'", strings.Join(args, sep)) +} + +// SSH executes ssh command with runSSHCommand as root. The `sudo` makes sure that all commands +// are executed by root, so that there won't be permission mismatch between different commands. +func (i *InstanceInfo) SSH(cmd ...string) (string, error) { + return runSSHCommand("ssh", append([]string{i.GetSSHTarget(), "--", "sudo"}, cmd...)...) +} + +func (i *InstanceInfo) CreateSSHTunnel(localPort, serverPort string) (int, error) { + args := []string{"-nNT", "-L", fmt.Sprintf("%s:localhost:%s", localPort, serverPort), i.GetSSHTarget()} + if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { + glog.V(4).Infof("Running on Jenkins, using special private key file at %v", pk) + args = append([]string{"-i", pk}, args...) + } else { + args = append([]string{"-i", sshDefaultKey}, args...) + } + args = append(strings.Split(sshOption, " "), args...) + cmd := exec.Command("ssh", args...) + err := cmd.Start() + if err != nil { + return 0, err + } + // TODO: use this process and kill it at the end of the test as well. + return cmd.Process.Pid, nil +} + +// SSHNoSudo executes ssh command with runSSHCommand as normal user. Sometimes we need this, +// for example creating a directory that we'll copy files there with scp. +func (i *InstanceInfo) SSHNoSudo(cmd ...string) (string, error) { + return runSSHCommand("ssh", append([]string{i.GetSSHTarget(), "--"}, cmd...)...) +} + +// SSHCheckAlive just pings the server quickly to check whether it is reachable by SSH +func (i *InstanceInfo) SSHCheckAlive() (string, error) { + return runSSHCommand("ssh", []string{i.GetSSHTarget(), "-o", "ConnectTimeout=10", "--", "echo"}...) +} + +// runSSHCommand executes the ssh or scp command, adding the flag provided --ssh-options +func runSSHCommand(cmd string, args ...string) (string, error) { + if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { + glog.V(4).Infof("Running on Jenkins, using special private key file at %v", pk) + args = append([]string{"-i", pk}, args...) + } else { + args = append([]string{"-i", sshDefaultKey}, args...) + } + args = append(strings.Split(sshOption, " "), args...) + + glog.V(4).Infof("Executing SSH command: %v %v", cmd, args) + + output, err := exec.Command(cmd, args...).CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err) + } + return string(output), nil +} diff --git a/test/e2e/gce_pd_e2e_test.go b/test/e2e/gce_pd_e2e_test.go deleted file mode 100644 index 0e9727657..000000000 --- a/test/e2e/gce_pd_e2e_test.go +++ /dev/null @@ -1,327 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "io/ioutil" - "net" - "os" - "path/filepath" - "testing" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/apimachinery/pkg/util/uuid" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager" - - csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" - gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider" - driver "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-pd-csi-driver" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -const ( - endpoint = "unix:/tmp/csi.sock" - addr = "/tmp/csi.sock" - network = "unix" - testNamePrefix = "gcepd-csi-e2e-" - - defaultSizeGb int64 = 5 - readyState = "READY" - standardDiskType = "pd-standard" - ssdDiskType = "pd-ssd" -) - -var ( - client *csiClient - gceCloud *gce.CloudProvider - nodeID string - - stdVolCap = &csipb.VolumeCapability{ - AccessType: &csipb.VolumeCapability_Mount{ - Mount: &csipb.VolumeCapability_MountVolume{}, - }, - AccessMode: &csipb.VolumeCapability_AccessMode{ - Mode: csipb.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - stdVolCaps = []*csipb.VolumeCapability{ - stdVolCap, - } -) - -func TestE2E(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests") -} - -var _ = BeforeSuite(func() { - var err error - // TODO(dyzz): better defaults - driverName := "testdriver" - nodeID = "gce-pd-csi-e2e" - vendorVersion := "testVendor" - - // TODO(dyzz): Start a driver - gceDriver := driver.GetGCEDriver() - gceCloud, err = gce.CreateCloudProvider(vendorVersion) - - Expect(err).To(BeNil(), "Failed to get cloud provider: %v", err) - - // TODO(dyzz): Change this to a fake mounter - mounter, err := mountmanager.CreateMounter() - - Expect(err).To(BeNil(), "Failed to get mounter %v", err) - - //Initialize GCE Driver - err = gceDriver.SetupGCEDriver(gceCloud, mounter, driverName, nodeID, vendorVersion) - Expect(err).To(BeNil(), "Failed to initialize GCE CSI Driver: %v", err) - - go func() { - gceDriver.Run(endpoint) - }() - - client = createCSIClient() - - Expect(err).To(BeNil(), "Failed to create cloud service") - // TODO: This is a hack to make sure the driver is fully up before running the tests, theres probably a better way to do this. - time.Sleep(20 * time.Second) -}) - -var _ = AfterSuite(func() { - // Close the client - err := client.conn.Close() - if err != nil { - Logf("Failed to close the client") - } else { - Logf("Closed the client") - } - // TODO(dyzz): Clean up driver and other things -}) - -var _ = Describe("GCE PD CSI Driver", func() { - - BeforeEach(func() { - err := client.assertCSIConnection() - Expect(err).To(BeNil(), "Failed to assert csi client connection: %v", err) - }) - - It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() { - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volId, err := CreateVolume(volName) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := gceCloud.GetDiskOrError(context.Background(), gceCloud.GetZone(), volName) - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - - defer func() { - // Delete Disk - DeleteVolume(volId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = gceCloud.GetDiskOrError(context.Background(), gceCloud.GetZone(), volName) - serverError, ok := status.FromError(err) - Expect(ok).To(BeTrue()) - Expect(serverError.Code()).To(Equal(codes.NotFound)) - }() - - // Attach Disk - err = ControllerPublishVolume(volId, nodeID) - Expect(err).To(BeNil(), "ControllerPublishVolume failed with error") - - defer func() { - // Detach Disk - err := ControllerUnpublishVolume(volId, nodeID) - Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error") - }() - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - NodeStageVolume(volId, stageDir) - Expect(err).To(BeNil(), "NodeStageVolume failed with error") - - defer func() { - // Unstage Disk - err := NodeUnstageVolume(volId, stageDir) - Expect(err).To(BeNil(), "NodeUnstageVolume failed with error") - }() - - // Mount Disk - publishDir := filepath.Join("/tmp/", volName, "mount") - err = NodePublishVolume(volId, stageDir, publishDir) - Expect(err).To(BeNil(), "NodePublishVolume failed with error") - - // Write a file - testFile := filepath.Join(publishDir, "testfile") - f, err := os.Create(testFile) - Expect(err).To(BeNil(), "Opening file %s failed with error", testFile) - - testString := "test-string" - f.WriteString(testString) - f.Sync() - f.Close() - - // Unmount Disk - err = NodeUnpublishVolume(volId, publishDir) - Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error") - - // Mount disk somewhere else - secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") - err = NodePublishVolume(volId, stageDir, secondPublishDir) - Expect(err).To(BeNil(), "NodePublishVolume failed with error") - - // Read File - fileContent, err := ioutil.ReadFile(filepath.Join(secondPublishDir, "testfile")) - Expect(err).To(BeNil(), "ReadFile failed with error") - Expect(string(fileContent)).To(Equal(testString)) - - // Unmount Disk - err = NodeUnpublishVolume(volId, secondPublishDir) - Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error") - - }) - - // Test volume already exists - - // Test volume with op pending -}) - -func Logf(format string, args ...interface{}) { - fmt.Fprint(GinkgoWriter, args...) -} - -type csiClient struct { - conn *grpc.ClientConn - idClient csipb.IdentityClient - nodeClient csipb.NodeClient - ctrlClient csipb.ControllerClient -} - -func createCSIClient() *csiClient { - return &csiClient{} -} - -func (c *csiClient) assertCSIConnection() error { - if c.conn == nil { - conn, err := grpc.Dial( - addr, - grpc.WithInsecure(), - grpc.WithDialer(func(target string, timeout time.Duration) (net.Conn, error) { - return net.Dial(network, target) - }), - ) - if err != nil { - return err - } - c.conn = conn - c.idClient = csipb.NewIdentityClient(conn) - c.nodeClient = csipb.NewNodeClient(conn) - c.ctrlClient = csipb.NewControllerClient(conn) - } - return nil -} - -func CreateVolume(volName string) (string, error) { - cvr := &csipb.CreateVolumeRequest{ - Name: volName, - VolumeCapabilities: stdVolCaps, - } - cresp, err := client.ctrlClient.CreateVolume(context.Background(), cvr) - if err != nil { - return "", err - } - return cresp.GetVolume().GetId(), nil -} - -func DeleteVolume(volId string) error { - dvr := &csipb.DeleteVolumeRequest{ - VolumeId: volId, - } - _, err := client.ctrlClient.DeleteVolume(context.Background(), dvr) - return err -} - -func ControllerPublishVolume(volId, nodeId string) error { - cpreq := &csipb.ControllerPublishVolumeRequest{ - VolumeId: volId, - NodeId: nodeId, - VolumeCapability: stdVolCap, - Readonly: false, - } - _, err := client.ctrlClient.ControllerPublishVolume(context.Background(), cpreq) - return err -} - -func ControllerUnpublishVolume(volId, nodeId string) error { - cupreq := &csipb.ControllerUnpublishVolumeRequest{ - VolumeId: volId, - NodeId: nodeId, - } - _, err := client.ctrlClient.ControllerUnpublishVolume(context.Background(), cupreq) - return err -} - -func NodeStageVolume(volId, stageDir string) error { - nodeStageReq := &csipb.NodeStageVolumeRequest{ - VolumeId: volId, - StagingTargetPath: stageDir, - VolumeCapability: stdVolCap, - } - _, err := client.nodeClient.NodeStageVolume(context.Background(), nodeStageReq) - return err -} - -func NodeUnstageVolume(volId, stageDir string) error { - nodeUnstageReq := &csipb.NodeUnstageVolumeRequest{ - VolumeId: volId, - StagingTargetPath: stageDir, - } - _, err := client.nodeClient.NodeUnstageVolume(context.Background(), nodeUnstageReq) - return err -} - -func NodeUnpublishVolume(volumeId, publishDir string) error { - nodeUnpublishReq := &csipb.NodeUnpublishVolumeRequest{ - VolumeId: volumeId, - TargetPath: publishDir, - } - _, err := client.nodeClient.NodeUnpublishVolume(context.Background(), nodeUnpublishReq) - return err -} - -func NodePublishVolume(volumeId, stageDir, publishDir string) error { - nodePublishReq := &csipb.NodePublishVolumeRequest{ - VolumeId: volumeId, - StagingTargetPath: stageDir, - TargetPath: publishDir, - VolumeCapability: stdVolCap, - Readonly: false, - } - _, err := client.nodeClient.NodePublishVolume(context.Background(), nodePublishReq) - return err -} diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go new file mode 100644 index 000000000..53bc7441b --- /dev/null +++ b/test/e2e/tests/setup_e2e_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "flag" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + project = flag.String("project", "", "Project to run tests in") + serviceAccount = flag.String("service-account", "", "Service account to bring up instance with") + runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") +) + +func TestE2E(t *testing.T) { + flag.Parse() + RegisterFailHandler(Fail) + RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests") +} diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go new file mode 100644 index 000000000..b46ecffaf --- /dev/null +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -0,0 +1,176 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "fmt" + "path/filepath" + "strings" + + "k8s.io/apimachinery/pkg/util/uuid" + remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + network = "unix" + testNamePrefix = "gcepd-csi-e2e-" + + defaultSizeGb int64 = 5 + readyState = "READY" + standardDiskType = "pd-standard" + ssdDiskType = "pd-ssd" +) + +var ( + client *utils.CsiClient + instance *remote.InstanceInfo + //gceCloud *gce.CloudProvider + nodeID string +) + +var _ = BeforeSuite(func() { + var err error + // TODO(dyzz): better defaults + nodeID = "gce-pd-csi-e2e-us-central1-c" + port := "2000" + if *runInProw { + *project, *serviceAccount = utils.SetupProwConfig() + } + + Expect(*project).ToNot(BeEmpty(), "Project should not be empty") + Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty") + + instance, err = utils.SetupInstanceAndDriver(*project, "us-central1-c", nodeID, port, *serviceAccount) + Expect(err).To(BeNil()) + + client = utils.CreateCSIClient(fmt.Sprintf("localhost:%s", port)) +}) + +var _ = AfterSuite(func() { + // Close the client + err := client.CloseConn() + if err != nil { + Logf("Failed to close the client") + } else { + Logf("Closed the client") + } + + // instance.DeleteInstance() +}) + +var _ = Describe("GCE PD CSI Driver", func() { + + BeforeEach(func() { + err := client.AssertCSIConnection() + Expect(err).To(BeNil(), "Failed to assert csi client connection: %v", err) + }) + + It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() { + // Create Disk + volName := testNamePrefix + string(uuid.NewUUID()) + volId, err := client.CreateVolume(volName) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // TODO: Validate Disk Created + /*cloudDisk, err := gceCloud.GetDiskOrError(context.Background(), gceCloud.GetZone(), volName) + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName))*/ + + defer func() { + // Delete Disk + client.DeleteVolume(volId) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // TODO: Validate Disk Deleted + /*_, err = gceCloud.GetDiskOrError(context.Background(), gceCloud.GetZone(), volName) + serverError, ok := status.FromError(err) + Expect(ok).To(BeTrue()) + Expect(serverError.Code()).To(Equal(codes.NotFound))*/ + }() + + // Attach Disk + err = client.ControllerPublishVolume(volId, nodeID) + Expect(err).To(BeNil(), "ControllerPublishVolume failed with error") + + defer func() { + // Detach Disk + err := client.ControllerUnpublishVolume(volId, nodeID) + Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error") + }() + + // Stage Disk + stageDir := filepath.Join("/tmp/", volName, "stage") + client.NodeStageVolume(volId, stageDir) + Expect(err).To(BeNil(), "NodeStageVolume failed with error") + + defer func() { + // Unstage Disk + err := client.NodeUnstageVolume(volId, stageDir) + Expect(err).To(BeNil(), "NodeUnstageVolume failed with error") + err = utils.RmAll(instance, filepath.Join("/tmp/", volName)) + Expect(err).To(BeNil(), "Failed to remove temp directory") + }() + + // Mount Disk + publishDir := filepath.Join("/tmp/", volName, "mount") + err = client.NodePublishVolume(volId, stageDir, publishDir) + Expect(err).To(BeNil(), "NodePublishVolume failed with error") + err = utils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") + Expect(err).To(BeNil(), "Chmod failed with error") + + // Write a file + testFileContents := "test" + testFile := filepath.Join(publishDir, "testfile") + err = utils.WriteFile(instance, testFile, testFileContents) + Expect(err).To(BeNil(), "Failed to write file") + + // Unmount Disk + err = client.NodeUnpublishVolume(volId, publishDir) + Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error") + + // Mount disk somewhere else + secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") + err = client.NodePublishVolume(volId, stageDir, secondPublishDir) + Expect(err).To(BeNil(), "NodePublishVolume failed with error") + err = utils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") + Expect(err).To(BeNil(), "Chmod failed with error") + + // Read File + secondTestFile := filepath.Join(secondPublishDir, "testfile") + readContents, err := utils.ReadFile(instance, secondTestFile) + Expect(err).To(BeNil(), "ReadFile failed with error") + Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents)) + + // Unmount Disk + err = client.NodeUnpublishVolume(volId, secondPublishDir) + Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error") + + }) + + // Test volume already exists + + // Test volume with op pending +}) + +func Logf(format string, args ...interface{}) { + fmt.Fprint(GinkgoWriter, args...) +} diff --git a/test/e2e/utils/client_wrappers.go b/test/e2e/utils/client_wrappers.go new file mode 100644 index 000000000..e7c81a175 --- /dev/null +++ b/test/e2e/utils/client_wrappers.go @@ -0,0 +1,168 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "time" + + csipb "github.com/container-storage-interface/spec/lib/go/csi/v0" + "github.com/golang/glog" + "google.golang.org/grpc" + + "k8s.io/apimachinery/pkg/util/wait" +) + +var ( + stdVolCap = &csipb.VolumeCapability{ + AccessType: &csipb.VolumeCapability_Mount{ + Mount: &csipb.VolumeCapability_MountVolume{}, + }, + AccessMode: &csipb.VolumeCapability_AccessMode{ + Mode: csipb.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + } + stdVolCaps = []*csipb.VolumeCapability{ + stdVolCap, + } +) + +type CsiClient struct { + conn *grpc.ClientConn + idClient csipb.IdentityClient + nodeClient csipb.NodeClient + ctrlClient csipb.ControllerClient + + endpoint string +} + +func CreateCSIClient(endpoint string) *CsiClient { + return &CsiClient{endpoint: endpoint} +} + +func (c *CsiClient) AssertCSIConnection() error { + var err error + + if err != nil { + return err + } + if c.conn == nil { + var conn *grpc.ClientConn + err = wait.Poll(10*time.Second, 3*time.Minute, func() (bool, error) { + conn, err = grpc.Dial( + c.endpoint, + grpc.WithInsecure(), + ) + if err != nil { + glog.Warningf("Client failed to dail endpoint %v", c.endpoint) + return false, nil + } + return true, nil + }) + if err != nil || conn == nil { + return fmt.Errorf("Failed to get client connection: %v", err) + } + c.conn = conn + c.idClient = csipb.NewIdentityClient(conn) + c.nodeClient = csipb.NewNodeClient(conn) + c.ctrlClient = csipb.NewControllerClient(conn) + } + return nil +} + +func (c *CsiClient) CloseConn() error { + return c.conn.Close() +} + +func (c *CsiClient) CreateVolume(volName string) (string, error) { + cvr := &csipb.CreateVolumeRequest{ + Name: volName, + VolumeCapabilities: stdVolCaps, + } + cresp, err := c.ctrlClient.CreateVolume(context.Background(), cvr) + if err != nil { + return "", err + } + return cresp.GetVolume().GetId(), nil +} + +func (c *CsiClient) DeleteVolume(volId string) error { + dvr := &csipb.DeleteVolumeRequest{ + VolumeId: volId, + } + _, err := c.ctrlClient.DeleteVolume(context.Background(), dvr) + return err +} + +func (c *CsiClient) ControllerPublishVolume(volId, nodeId string) error { + cpreq := &csipb.ControllerPublishVolumeRequest{ + VolumeId: volId, + NodeId: nodeId, + VolumeCapability: stdVolCap, + Readonly: false, + } + _, err := c.ctrlClient.ControllerPublishVolume(context.Background(), cpreq) + return err +} + +func (c *CsiClient) ControllerUnpublishVolume(volId, nodeId string) error { + cupreq := &csipb.ControllerUnpublishVolumeRequest{ + VolumeId: volId, + NodeId: nodeId, + } + _, err := c.ctrlClient.ControllerUnpublishVolume(context.Background(), cupreq) + return err +} + +func (c *CsiClient) NodeStageVolume(volId, stageDir string) error { + nodeStageReq := &csipb.NodeStageVolumeRequest{ + VolumeId: volId, + StagingTargetPath: stageDir, + VolumeCapability: stdVolCap, + } + _, err := c.nodeClient.NodeStageVolume(context.Background(), nodeStageReq) + return err +} + +func (c *CsiClient) NodeUnstageVolume(volId, stageDir string) error { + nodeUnstageReq := &csipb.NodeUnstageVolumeRequest{ + VolumeId: volId, + StagingTargetPath: stageDir, + } + _, err := c.nodeClient.NodeUnstageVolume(context.Background(), nodeUnstageReq) + return err +} + +func (c *CsiClient) NodeUnpublishVolume(volumeId, publishDir string) error { + nodeUnpublishReq := &csipb.NodeUnpublishVolumeRequest{ + VolumeId: volumeId, + TargetPath: publishDir, + } + _, err := c.nodeClient.NodeUnpublishVolume(context.Background(), nodeUnpublishReq) + return err +} + +func (c *CsiClient) NodePublishVolume(volumeId, stageDir, publishDir string) error { + nodePublishReq := &csipb.NodePublishVolumeRequest{ + VolumeId: volumeId, + StagingTargetPath: stageDir, + TargetPath: publishDir, + VolumeCapability: stdVolCap, + Readonly: false, + } + _, err := c.nodeClient.NodePublishVolume(context.Background(), nodePublishReq) + return err +} diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go new file mode 100644 index 000000000..edc539b8c --- /dev/null +++ b/test/e2e/utils/utils.go @@ -0,0 +1,180 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "os" + "path" + "time" + + "github.com/golang/glog" + "golang.org/x/oauth2/google" + cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" + boskosclient "k8s.io/test-infra/boskos/client" + remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote" +) + +var ( + boskos = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos") +) + +const ( + archiveName = "e2e_gce_pd_test.tar.gz" +) + +func SetupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, instanceServiceAccount string) (*remote.InstanceInfo, error) { + // Create the instance in the requisite zone + instance, err := remote.CreateInstanceInfo(instanceProject, instanceZone, instanceName) + if err != nil { + return nil, err + } + + err = instance.CreateInstance(instanceServiceAccount) + + if err != nil { + return nil, err + } + + // Create Driver Archive + + goPath, ok := os.LookupEnv("GOPATH") + if !ok { + return nil, fmt.Errorf("Could not find environment variable GOPATH") + } + pkgPath := path.Join(goPath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/") + binPath := path.Join(pkgPath, "bin/gce-pd-csi-driver") + archivePath, err := remote.CreateDriverArchive(archiveName, pkgPath, binPath) + if err != nil { + return nil, err + } + defer func() { + err = os.Remove(archivePath) + if err != nil { + glog.Warningf("Failed to remove archive file %s: %v", archivePath, err) + } + }() + + // Upload archive to instance and run binaries + endpoint := fmt.Sprintf("tcp://localhost:%s", port) + workspace := remote.NewWorkspaceDir("gce-pd-e2e-") + driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver --endpoint=%s --nodeid=%s > %s/prog.out 2> %s/prog.err < /dev/null &'", + workspace, endpoint, instanceName, workspace, workspace) + err = instance.UploadAndRun(archivePath, workspace, driverRunCmd) + if err != nil { + return nil, err + } + + // Create an SSH tunnel from port to port + res, err := instance.CreateSSHTunnel(port, port) + if err != nil { + return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", res, err) + } + + return instance, nil +} + +func SetupProwConfig() (project, serviceAccount string) { + // Try to get a Boskos project + glog.V(4).Infof("Running in PROW") + glog.V(4).Infof("Fetching a Boskos loaned project") + + p, err := boskos.Acquire("gce-project", "free", "busy") + if err != nil { + glog.Fatal("boskos failed to acquire project: %v", err) + } + + if p == nil { + glog.Fatal("boskos does not have a free gce-project at the moment") + } + + project = p.GetName() + + go func(c *boskosclient.Client, proj string) { + for range time.Tick(time.Minute * 5) { + if err := c.UpdateOne(p.Name, "busy", nil); err != nil { + glog.Warningf("[Boskos] Update %s failed with %v", p, err) + } + } + }(boskos, p.Name) + + // If we're on CI overwrite the service account + glog.V(4).Infof("Fetching the default compute service account") + + c, err := google.DefaultClient(context.TODO(), cloudresourcemanager.CloudPlatformScope) + if err != nil { + glog.Fatalf("Failed to get Google Default Client: %v", err) + } + + cloudresourcemanagerService, err := cloudresourcemanager.New(c) + if err != nil { + glog.Fatalf("Failed to create new cloudresourcemanager: %v", err) + } + + resp, err := cloudresourcemanagerService.Projects.Get(project).Do() + if err != nil { + glog.Fatal("Failed to get project %v from Cloud Resource Manager: %v", project, err) + } + + // Default Compute Engine service account + // [PROJECT_NUMBER]-compute@developer.gserviceaccount.com + serviceAccount = fmt.Sprintf("%v-compute@developer.gserviceaccount.com", resp.ProjectNumber) + return project, serviceAccount +} + +func ForceChmod(instance *remote.InstanceInfo, filePath string, perms string) error { + originalumask, err := instance.SSHNoSudo("umask") + if err != nil { + return fmt.Errorf("failed to umask. Output: %v, errror: %v", originalumask, err) + } + output, err := instance.SSHNoSudo("umask", "0000") + if err != nil { + return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err) + } + output, err = instance.SSH("chmod", "-R", perms, filePath) + if err != nil { + return fmt.Errorf("failed to chmod file %s. Output: %v, errror: %v", filePath, output, err) + } + output, err = instance.SSHNoSudo("umask", originalumask) + if err != nil { + return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err) + } + return nil +} + +func WriteFile(instance *remote.InstanceInfo, filePath, fileContents string) error { + output, err := instance.SSHNoSudo("echo", fileContents, ">", filePath) + if err != nil { + return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", filePath, output, err) + } + return nil +} + +func ReadFile(instance *remote.InstanceInfo, filePath string) (string, error) { + output, err := instance.SSHNoSudo("cat", filePath) + if err != nil { + return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", filePath, output, err) + } + return output, nil +} + +func RmAll(instance *remote.InstanceInfo, filePath string) error { + output, err := instance.SSH("rm", "-rf", filePath) + if err != nil { + return fmt.Errorf("failed to delete all %s. Output: %v, errror: %v", filePath, output, err) + } + return nil +} diff --git a/test/remote/remote/e2e.go b/test/remote/remote/e2e.go deleted file mode 100644 index dba5ac620..000000000 --- a/test/remote/remote/e2e.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2016 Google - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/golang/glog" -) - -// E2ERemote is type for GCE PD CSI Driver Remote E2E Tests -type E2ERemote struct{} - -// InitE2ERemote initializes the GCE PD CSI Driver remote E2E suite -func InitE2ERemote() TestSuite { - return &E2ERemote{} -} - -// SetupTestPackage sets up the test package with binaries k8s required for node e2e tests -func (n *E2ERemote) SetupTestPackage(tardir string) error { - // TODO(dyzz): build the gce driver tests instead. - - cmd := exec.Command("go", "test", "-c", "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e", "-o", "test/e2e/e2e.test") - err := cmd.Run() - - if err != nil { - return fmt.Errorf("Failed to build test: %v", err) - } - - cmd = exec.Command("mkdir", "-p", "bin") - err = cmd.Run() - if err != nil { - return fmt.Errorf("Failed to mkdir bin/: %v", err) - } - - cmd = exec.Command("cp", "test/e2e/e2e.test", "bin/e2e.test") - err = cmd.Run() - if err != nil { - return fmt.Errorf("Failed to copy: %v", err) - } - // Copy binaries - requiredBins := []string{"e2e.test"} - for _, bin := range requiredBins { - source := filepath.Join("bin", bin) - if _, err := os.Stat(source); err != nil { - return fmt.Errorf("failed to locate test binary %s: %v", bin, err) - } - out, err := exec.Command("cp", source, filepath.Join(tardir, bin)).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to copy %q: %v Output: %q", bin, err, out) - } - } - - return nil -} - -// RunTest runs test on the node. -func (n *E2ERemote) RunTest(host, workspace, results, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) { - glog.V(2).Infof("Starting tests on %q", host) - cmd := getSSHCommand(" && ", - fmt.Sprintf("cd %s", workspace), - fmt.Sprintf("./e2e.test"), - ) - return SSH(host, "sh", "-c", cmd) -} diff --git a/test/remote/remote/remote.go b/test/remote/remote/remote.go deleted file mode 100644 index 888150341..000000000 --- a/test/remote/remote/remote.go +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/golang/glog" - utilerrors "k8s.io/apimachinery/pkg/util/errors" -) - -var testTimeoutSeconds = flag.Duration("test-timeout", 45*time.Minute, "How long (in golang duration format) to wait for ginkgo tests to complete.") -var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.") - -const archiveName = "e2e_gce_pd_test.tar.gz" - -func CreateTestArchive(suite TestSuite) (string, error) { - glog.V(2).Infof("Building archive...") - tardir, err := ioutil.TempDir("", "gce-pd-e2e-archive") - if err != nil { - return "", fmt.Errorf("failed to create temporary directory %v", err) - } - defer os.RemoveAll(tardir) - - // Call the suite function to setup the test package. - err = suite.SetupTestPackage(tardir) - if err != nil { - return "", fmt.Errorf("failed to setup test package %q: %v", tardir, err) - } - - // Build the tar - out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err, out) - } - - dir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("failed to get working directory %v", err) - } - return filepath.Join(dir, archiveName), nil -} - -// RunRemote returns the command output, whether the exit was ok, and any errors -func RunRemote(suite TestSuite, archive string, host string, cleanup bool, testArgs string, ginkgoArgs string) (string, bool, error) { - // Create the temp staging directory - glog.V(2).Infof("Staging test binaries on %q", host) - workspace := newWorkspaceDir() - // Do not sudo here, so that we can use scp to copy test archive to the directdory. - if output, err := SSHNoSudo(host, "mkdir", workspace); err != nil { - // Exit failure with the error - return "", false, fmt.Errorf("failed to create workspace directory %q on host %q: %v output: %q", workspace, host, err, output) - } - if cleanup { - defer func() { - output, err := SSH(host, "rm", "-rf", workspace) - if err != nil { - glog.Errorf("failed to cleanup workspace %q on host %q: %v. Output:\n%s", workspace, host, err, output) - } - }() - } - - // Copy the archive to the staging directory - if output, err := runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIP(host), workspace)); err != nil { - // Exit failure with the error - return "", false, fmt.Errorf("failed to copy test archive: %v, output: %q", err, output) - } - - // Extract the archive - cmd := getSSHCommand(" && ", - fmt.Sprintf("cd %s", workspace), - fmt.Sprintf("tar -xzvf ./%s", archiveName), - ) - glog.V(2).Infof("Extracting tar on %q", host) - // Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but - // we want the extracted files to be owned by the current user. - if output, err := SSHNoSudo(host, "sh", "-c", cmd); err != nil { - // Exit failure with the error - return "", false, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output) - } - - // Create the test result directory. - resultDir := filepath.Join(workspace, "results") - if output, err := SSHNoSudo(host, "mkdir", resultDir); err != nil { - // Exit failure with the error - return "", false, fmt.Errorf("failed to create test result directory %q on host %q: %v output: %q", resultDir, host, err, output) - } - - glog.V(2).Infof("Running test on %q", host) - output, err := suite.RunTest(host, workspace, resultDir, testArgs, ginkgoArgs, *testTimeoutSeconds) - - aggErrs := []error{} - // Do not log the output here, let the caller deal with the test output. - if err != nil { - aggErrs = append(aggErrs, err) - } - - /* - glog.V(2).Infof("Copying test artifacts from %q", host) - scpErr := getTestArtifacts(host, workspace) - if scpErr != nil { - aggErrs = append(aggErrs, scpErr) - } - */ - - return output, len(aggErrs) == 0, utilerrors.NewAggregate(aggErrs) -} - -const ( - // workspaceDirPrefix is the string prefix used in the workspace directory name. - workspaceDirPrefix = "gce-pd-e2e-" - // timestampFormat is the timestamp format used in the e2e directory name. - timestampFormat = "20060102T150405" -) - -func getTimestamp() string { - return fmt.Sprintf(time.Now().Format(timestampFormat)) -} - -func newWorkspaceDir() string { - return filepath.Join("/tmp", workspaceDirPrefix+getTimestamp()) -} - -// Parses the workspace directory name and gets the timestamp part of it. -// This can later be used to name other artifacts (such as the -// kubelet-${instance}.service systemd transient service used to launch -// Kubelet) so that they can be matched to each other. -func GetTimestampFromWorkspaceDir(dir string) string { - dirTimestamp := strings.TrimPrefix(filepath.Base(dir), workspaceDirPrefix) - re := regexp.MustCompile("^\\d{8}T\\d{6}$") - if re.MatchString(dirTimestamp) { - return dirTimestamp - } - // Fallback: if we can't find that timestamp, default to using Now() - return getTimestamp() -} - -func getTestArtifacts(host, testDir string) error { - logPath := filepath.Join(*resultsDir, host) - if err := os.MkdirAll(logPath, 0755); err != nil { - return fmt.Errorf("failed to create log directory %q: %v", logPath, err) - } - // Copy logs to artifacts/hostname - if _, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.log", GetHostnameOrIP(host), testDir), logPath); err != nil { - return err - } - // Copy json files (if any) to artifacts. - if _, err := SSH(host, "ls", fmt.Sprintf("%s/results/*.json", testDir)); err == nil { - if _, err = runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.json", GetHostnameOrIP(host), testDir), *resultsDir); err != nil { - return err - } - } - if _, err := SSH(host, "ls", fmt.Sprintf("%s/results/junit*", testDir)); err == nil { - // Copy junit (if any) to the top of artifacts - if _, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIP(host), testDir), *resultsDir); err != nil { - return err - } - } - return nil -} - -// WriteLog is a temporary function to make it possible to write log -// in the runner. This is used to collect serial console log. -// TODO(random-liu): Use the log-dump script in cluster e2e. -func WriteLog(host, filename, content string) error { - logPath := filepath.Join(*resultsDir, host) - if err := os.MkdirAll(logPath, 0755); err != nil { - return fmt.Errorf("failed to create log directory %q: %v", logPath, err) - } - f, err := os.Create(filepath.Join(logPath, filename)) - if err != nil { - return err - } - defer f.Close() - _, err = f.WriteString(content) - return err -} diff --git a/test/remote/remote/ssh.go b/test/remote/remote/ssh.go deleted file mode 100644 index 9decb4e78..000000000 --- a/test/remote/remote/ssh.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "flag" - "fmt" - "os" - "os/exec" - "os/user" - "strings" - "sync" - - "github.com/golang/glog" -) - -var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") -var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce") -var sshKey = flag.String("ssh-key", "", "Path to ssh private key.") -var sshUser = flag.String("ssh-user", "", "Use predefined user for ssh.") - -var sshOptionsMap map[string]string -var sshDefaultKeyMap map[string]string - -func init() { - usr, err := user.Current() - if err != nil { - glog.Fatal(err) - } - sshOptionsMap = map[string]string{ - "gce": "-o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o LogLevel=ERROR", - } - sshDefaultKeyMap = map[string]string{ - "gce": fmt.Sprintf("%s/.ssh/google_compute_engine", usr.HomeDir), - } -} - -var hostnameIpOverrides = struct { - sync.RWMutex - m map[string]string -}{m: make(map[string]string)} - -func AddHostnameIP(hostname, ip string) { - hostnameIpOverrides.Lock() - defer hostnameIpOverrides.Unlock() - hostnameIpOverrides.m[hostname] = ip -} - -// GetHostnameOrIP converts hostname into ip and apply user if necessary. -func GetHostnameOrIP(hostname string) string { - hostnameIpOverrides.RLock() - defer hostnameIpOverrides.RUnlock() - host := hostname - if ip, found := hostnameIpOverrides.m[hostname]; found { - host = ip - } - if *sshUser != "" { - host = fmt.Sprintf("%s@%s", *sshUser, host) - } else if _, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { - host = fmt.Sprintf("prow@%s", host) - } - return host -} - -// getSSHCommand handles proper quoting so that multiple commands are executed in the same shell over ssh -func getSSHCommand(sep string, args ...string) string { - return fmt.Sprintf("'%s'", strings.Join(args, sep)) -} - -// SSH executes ssh command with runSSHCommand as root. The `sudo` makes sure that all commands -// are executed by root, so that there won't be permission mismatch between different commands. -func SSH(host string, cmd ...string) (string, error) { - return runSSHCommand("ssh", append([]string{GetHostnameOrIP(host), "--", "sudo"}, cmd...)...) -} - -// SSHNoSudo executes ssh command with runSSHCommand as normal user. Sometimes we need this, -// for example creating a directory that we'll copy files there with scp. -func SSHNoSudo(host string, cmd ...string) (string, error) { - return runSSHCommand("ssh", append([]string{GetHostnameOrIP(host), "--"}, cmd...)...) -} - -// SSHCheckAlive just pings the server quickly to check whether it is reachable by SSH -func SSHCheckAlive(host string) (string, error) { - return runSSHCommand("ssh", []string{GetHostnameOrIP(host), "-o", "ConnectTimeout=10", "--", "echo"}...) -} - -// runSSHCommand executes the ssh or scp command, adding the flag provided --ssh-options -func runSSHCommand(cmd string, args ...string) (string, error) { - if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { - glog.V(4).Infof("Running on Jenkins, using special private key file at %v", pk) - args = append([]string{"-i", pk}, args...) - } else if *sshKey != "" { - args = append([]string{"-i", *sshKey}, args...) - } else if key, found := sshDefaultKeyMap[*sshEnv]; found { - args = append([]string{"-i", key}, args...) - } - if env, found := sshOptionsMap[*sshEnv]; found { - args = append(strings.Split(env, " "), args...) - } - if *sshOptions != "" { - args = append(strings.Split(*sshOptions, " "), args...) - } - output, err := exec.Command(cmd, args...).CombinedOutput() - if err != nil { - return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err) - } - return string(output), nil -} diff --git a/test/remote/remote/types.go b/test/remote/remote/types.go deleted file mode 100644 index e6c272551..000000000 --- a/test/remote/remote/types.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package remote contains implementations of the TestSuite interface -package remote - -import ( - "time" -) - -// TestSuite is the interface of a test suite -type TestSuite interface { - // SetupTestPackage setup the test package in the given directory. TestSuite - // should put all necessary binaries and dependencies into the path. The caller - // will: - // * create a tarball with the directory. - // * deploy the tarball to the testing host. - // * untar the tarball to the testing workspace on the testing host. - SetupTestPackage(path string) error - // RunTest runs test on the node in the given workspace and returns test output - // and test error if there is any. - // * host is the target node to run the test. - // * workspace is the directory on the testing host the test is running in. Note - // that the test package is unpacked in the workspace before running the test. - // * results is the directory the test should write result into. All logs should be - // saved as *.log, all junit file should start with junit*. - // * imageDesc is the description of the image the test is running on. - // It will be used for logging purpose only. - // * testArgs is the arguments passed to test. - // * ginkgoArgs is the arguments passed to ginkgo. - // * systemSpecName is the name of the system spec used for validating the - // image on which the test runs. - // * timeout is the test timeout. - RunTest(host, workspace, results, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) -} diff --git a/test/remote/run_remote/run_remote.go b/test/remote/run_remote/run_remote.go deleted file mode 100644 index 1f94810a1..000000000 --- a/test/remote/run_remote/run_remote.go +++ /dev/null @@ -1,560 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "flag" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "strings" - "sync" - "time" - - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/test-infra/boskos/client" - gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote/remote" - - "github.com/golang/glog" - "golang.org/x/oauth2/google" - "google.golang.org/api/cloudresourcemanager/v1" - compute "google.golang.org/api/compute/v0.beta" -) - -var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") -var zone = flag.String("zone", "", "gce zone the hosts live in") -var project = flag.String("project", "", "gce project the hosts live in") -var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances") -var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created") -var buildOnly = flag.Bool("build-only", false, "If true, build e2e_gce_pd_test.tar.gz and exit.") -var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.") -var serviceAccount = flag.String("service-account", "", "GCP Service Account to start the test instance under") -var runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") - -// envs is the type used to collect all node envs. The key is the env name, -// and the value is the env value -type envs map[string]string - -// String function of flag.Value -func (e *envs) String() string { - return fmt.Sprint(*e) -} - -// Set function of flag.Value -func (e *envs) Set(value string) error { - kv := strings.SplitN(value, "=", 2) - if len(kv) != 2 { - return fmt.Errorf("invalid env string") - } - emap := *e - emap[kv[0]] = kv[1] - return nil -} - -// nodeEnvs is the node envs from the flag `node-env`. -var nodeEnvs = make(envs) - -func init() { - flag.Var(&nodeEnvs, "node-env", "An environment variable passed to instance as metadata, e.g. when '--node-env=PATH=/usr/bin' is specified, there will be an extra instance metadata 'PATH=/usr/bin'.") -} - -const ( - defaultMachine = "n1-standard-1" - defaultFirewallRule = "default-allow-ssh" -) - -var ( - computeService *compute.Service - arc Archive - suite remote.TestSuite - - boskos = client.NewClient(os.Getenv("JOB_NAME"), "http://boskos") -) - -// Archive contains information about the test tar -type Archive struct { - sync.Once - path string - err error -} - -// TestResult contains info about results of test -type TestResult struct { - output string - err error - host string - exitOk bool -} - -func main() { - flag.Parse() - suite = remote.InitE2ERemote() - - if *runInProw { - // Try to get a Boskos project - glog.V(4).Infof("Running in PROW") - glog.V(4).Infof("Fetching a Boskos loaned project") - - p, err := boskos.Acquire("gce-project", "free", "busy") - if err != nil { - glog.Fatal("boskos failed to acquire project: %v", err) - } - - if p == nil { - glog.Fatal("boskos does not have a free gce-project at the moment") - } - - glog.Infof("Overwriting supplied project %v with project from Boskos: %v", *project, p.GetName()) - - *project = p.GetName() - - go func(c *client.Client, proj string) { - for range time.Tick(time.Minute * 5) { - if err := c.UpdateOne(p.Name, "busy", nil); err != nil { - glog.Warningf("[Boskos] Update %s failed with %v", p, err) - } - } - }(boskos, p.Name) - - // If we're on CI overwrite the service account - glog.V(4).Infof("Fetching the default compute service account") - - c, err := google.DefaultClient(context.TODO(), cloudresourcemanager.CloudPlatformScope) - if err != nil { - glog.Fatalf("Failed to get Google Default Client: %v", err) - } - - cloudresourcemanagerService, err := cloudresourcemanager.New(c) - if err != nil { - glog.Fatalf("Failed to create new cloudresourcemanager: %v", err) - } - - resp, err := cloudresourcemanagerService.Projects.Get(*project).Do() - if err != nil { - glog.Fatal("Failed to get project %v from Cloud Resource Manager: %v", *project, err) - } - - // Default Compute Engine service account - // [PROJECT_NUMBER]-compute@developer.gserviceaccount.com - sa := fmt.Sprintf("%v-compute@developer.gserviceaccount.com", resp.ProjectNumber) - glog.Infof("Overwriting supplied service account %v with PROW service account %v", *serviceAccount, sa) - - *serviceAccount = sa - } - - if *project == "" { - glog.Fatal("Project must be speficied") - } - - if *zone == "" { - glog.Fatal("Zone must be specified") - } - - if *serviceAccount == "" { - glog.Fatal("You must specify a service account to create an instance under that has at least OWNERS permissions on disks and READER on instances.") - } - - rand.Seed(time.Now().UTC().UnixNano()) - if *buildOnly { - // Build the archive and exit - remote.CreateTestArchive(suite) - return - } - - var err error - computeService, err = getComputeClient() - if err != nil { - glog.Fatalf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err) - } - - // Setup coloring - stat, _ := os.Stdout.Stat() - useColor := (stat.Mode() & os.ModeCharDevice) != 0 - blue := "" - noColour := "" - if useColor { - blue = "\033[0;34m" - noColour = "\033[0m" - } - - go arc.getArchive() - defer arc.deleteArchive() - - fmt.Printf("Initializing e2e tests") - results := test([]string{"TODO tests"}) - // Wait for all tests to complete and emit the results - errCount := 0 - host := results.host - fmt.Println() // Print an empty line - fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour) - fmt.Printf("%s> START TEST >%s\n", blue, noColour) - fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour) - fmt.Printf("Start Test Suite on Host %s\n", host) - fmt.Printf("%s\n", results.output) - if results.err != nil { - errCount++ - fmt.Printf("Failure Finished Test Suite on Host %s\n%v\n", host, results.err) - } else { - fmt.Printf("Success Finished Test Suite on Host %s\n", host) - } - fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour) - fmt.Printf("%s< FINISH TEST <%s\n", blue, noColour) - fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour) - fmt.Println() // Print an empty line - - if boskos.HasResource() { - if berr := boskos.ReleaseAll("dirty"); berr != nil { - glog.Fatalf("[Boskos] Fail To Release: %v, kubetest err: %v", berr, err) - } - } - - // Set the exit code if there were failures - if !results.exitOk { - fmt.Printf("Failure: %d errors encountered.\n", errCount) - arc.deleteArchive() - os.Exit(1) - } -} - -func (a *Archive) getArchive() (string, error) { - a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite) }) - return a.path, a.err -} - -func (a *Archive) deleteArchive() { - path, err := a.getArchive() - if err != nil { - return - } - os.Remove(path) -} - -// Run tests in archive against host -func testHost(host string, deleteFiles bool, ginkgoFlagsStr string) *TestResult { - instance, err := computeService.Instances.Get(*project, *zone, host).Do() - if err != nil { - return &TestResult{ - err: err, - host: host, - exitOk: false, - } - } - if strings.ToUpper(instance.Status) != "RUNNING" { - err = fmt.Errorf("instance %s not in state RUNNING, was %s", host, instance.Status) - return &TestResult{ - err: err, - host: host, - exitOk: false, - } - } - externalIP := getexternalIP(instance) - if len(externalIP) > 0 { - remote.AddHostnameIP(host, externalIP) - } - - path, err := arc.getArchive() - if err != nil { - // Don't log fatal because we need to do any needed cleanup contained in "defer" statements - return &TestResult{ - err: fmt.Errorf("unable to create test archive: %v", err), - } - } - - output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, *testArgs, ginkgoFlagsStr) - return &TestResult{ - output: output, - err: err, - host: host, - exitOk: exitOk, - } -} - -// Provision a gce instance using image and run the tests in archive against the instance. -// Delete the instance afterward. -func test(tests []string) *TestResult { - ginkgoFlagsStr := *ginkgoFlags - // Check whether the test is for benchmark. - if len(tests) > 0 { - // Use the Ginkgo focus in benchmark config. - ginkgoFlagsStr += (" " + testsToGinkgoFocus(tests)) - } - - host, err := createInstance(*serviceAccount) - if *deleteInstances { - defer deleteInstance(host) - } - if err != nil { - return &TestResult{ - err: fmt.Errorf("unable to create gce instance with running docker daemon for image. %v", err), - } - } - - // Only delete the files if we are keeping the instance and want it cleaned up. - // If we are going to delete the instance, don't bother with cleaning up the files - deleteFiles := !*deleteInstances && *cleanup - - result := testHost(host, deleteFiles, ginkgoFlagsStr) - // This is a temporary solution to collect serial node serial log. Only port 1 contains useful information. - // TODO(random-liu): Extract out and unify log collection logic with cluste e2e. - serialPortOutput, err := computeService.Instances.GetSerialPortOutput(*project, *zone, host).Port(1).Do() - if err != nil { - glog.Errorf("Failed to collect serial output from node %q: %v", host, err) - } else { - logFilename := "serial-1.log" - err := remote.WriteLog(host, logFilename, serialPortOutput.Contents) - if err != nil { - glog.Errorf("Failed to write serial output from node %q to %q: %v", host, logFilename, err) - } - } - return result -} - -// Create default SSH filewall rule if it does not exist -func createDefaultFirewallRule() error { - var err error - if _, err = computeService.Firewalls.Get(*project, defaultFirewallRule).Do(); err != nil { - glog.Infof("Default firewall rule %v does not exist, creating", defaultFirewallRule) - f := &compute.Firewall{ - Name: defaultFirewallRule, - Allowed: []*compute.FirewallAllowed{ - { - IPProtocol: "tcp", - Ports: []string{"22"}, - }, - }, - } - _, err = computeService.Firewalls.Insert(*project, f).Do() - if err != nil { - return fmt.Errorf("Failed to insert required default SSH firewall Rule %v: %v", defaultFirewallRule, err) - } - } else { - glog.Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) - } - return nil -} - -// Provision a gce instance using image -func createInstance(serviceAccount string) (string, error) { - var err error - - name := "gce-pd-csi-e2e" - myuuid := string(uuid.NewUUID()) - - err = createDefaultFirewallRule() - if err != nil { - return "", fmt.Errorf("Failed to create firewall rule: %v", err) - } - - glog.V(4).Infof("Creating instance: %v", name) - - // TODO: Pick a better boot disk image - imageURL := "projects/ml-images/global/images/family/tf-1-9" - i := &compute.Instance{ - Name: name, - MachineType: machineType(""), - NetworkInterfaces: []*compute.NetworkInterface{ - { - AccessConfigs: []*compute.AccessConfig{ - { - Type: "ONE_TO_ONE_NAT", - Name: "External NAT", - }, - }}, - }, - Disks: []*compute.AttachedDisk{ - { - AutoDelete: true, - Boot: true, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: "my-root-pd-" + myuuid, - SourceImage: imageURL, - }, - }, - }, - } - - saObj := &compute.ServiceAccount{ - Email: serviceAccount, - Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"}, - } - i.ServiceAccounts = []*compute.ServiceAccount{saObj} - - if pubkey, ok := os.LookupEnv("JENKINS_GCE_SSH_PUBLIC_KEY_FILE"); ok { - glog.V(4).Infof("JENKINS_GCE_SSH_PUBLIC_KEY_FILE set to %v, adding public key to Instance", pubkey) - meta, err := generateMetadataWithPublicKey(pubkey) - if err != nil { - return "", err - } - i.Metadata = meta - } - - if _, err := computeService.Instances.Get(*project, *zone, i.Name).Do(); err != nil { - op, err := computeService.Instances.Insert(*project, *zone, i).Do() - glog.V(4).Infof("Inserted instance %v in project %v, zone %v", i.Name, *project, *zone) - if err != nil { - ret := fmt.Sprintf("could not create instance %s: API error: %v", name, err) - if op != nil { - ret = fmt.Sprintf("%s: %v", ret, op.Error) - } - return "", fmt.Errorf(ret) - } else if op.Error != nil { - return "", fmt.Errorf("could not create instance %s: %+v", name, op.Error) - } - } else { - glog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", i.Name) - } - - then := time.Now() - err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - glog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", name, time.Since(then)) - var instance *compute.Instance - instance, err = computeService.Instances.Get(*project, *zone, name).Do() - if err != nil { - glog.Errorf("Failed to get instance %v: %v", name, err) - return false, nil - } - - if strings.ToUpper(instance.Status) != "RUNNING" { - glog.Warningf("instance %s not in state RUNNING, was %s", name, instance.Status) - return false, nil - } - - externalIP := getexternalIP(instance) - if len(externalIP) > 0 { - remote.AddHostnameIP(name, externalIP) - } - - if sshOut, err := remote.SSHCheckAlive(name); err != nil { - err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", name, err) - glog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut) - return false, nil - } - glog.Infof("Instance %v in state RUNNING and vailable by SSH", name) - return true, nil - }) - - // If instance didn't reach running state in time, return with error now. - if err != nil { - return name, err - } - // Instance reached running state in time, make sure that cloud-init is complete - glog.V(2).Infof("Instance %v has been created successfully", name) - return name, nil -} - -func generateMetadataWithPublicKey(pubKeyFile string) (*compute.Metadata, error) { - publicKeyByte, err := ioutil.ReadFile(pubKeyFile) - if err != nil { - return nil, err - } - - publicKey := string(publicKeyByte) - - // Take username and prepend it to the public key - tokens := strings.Split(publicKey, " ") - if len(tokens) != 3 { - return nil, fmt.Errorf("Public key not comprised of 3 parts, instead was: %v", publicKey) - } - publicKey = strings.TrimSpace(tokens[2]) + ":" + publicKey - newMeta := &compute.Metadata{ - Items: []*compute.MetadataItems{ - { - Key: "ssh-keys", - Value: &publicKey, - }, - }, - } - return newMeta, nil -} - -func getexternalIP(instance *compute.Instance) string { - for i := range instance.NetworkInterfaces { - ni := instance.NetworkInterfaces[i] - for j := range ni.AccessConfigs { - ac := ni.AccessConfigs[j] - if len(ac.NatIP) > 0 { - return ac.NatIP - } - } - } - return "" -} - -func getComputeClient() (*compute.Service, error) { - const retries = 10 - const backoff = time.Second * 6 - - // Setup the gce client for provisioning instances - // Getting credentials on gce jenkins is flaky, so try a couple times - var err error - var cs *compute.Service - for i := 0; i < retries; i++ { - if i > 0 { - time.Sleep(backoff) - } - - var client *http.Client - client, err = google.DefaultClient(context.TODO(), compute.ComputeScope) - if err != nil { - continue - } - - cs, err = compute.New(client) - if err != nil { - continue - } - return cs, nil - } - return nil, err -} - -func deleteInstance(host string) { - glog.V(4).Infof("Deleting instance %q", host) - _, err := computeService.Instances.Delete(*project, *zone, host).Do() - if err != nil { - if gce.IsGCEError(err, "notFound") { - return - } - glog.Errorf("Error deleting instance %q: %v", host, err) - } -} - -func machineType(machine string) string { - if machine == "" { - machine = defaultMachine - } - return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine) -} - -// testsToGinkgoFocus converts the test string list to Ginkgo focus -func testsToGinkgoFocus(tests []string) string { - focus := "--focus=\"" - for i, test := range tests { - if i == 0 { - focus += test - } else { - focus += ("|" + test) - } - } - return focus + "\"" -} diff --git a/test/run-e2e-local.sh b/test/run-e2e-local.sh index d75a438a2..58cfbef09 100755 --- a/test/run-e2e-local.sh +++ b/test/run-e2e-local.sh @@ -1,8 +1,8 @@ #!/bin/bash -set -e -set -x +set -o nounset +set -o errexit readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -go run "$GOPATH/src/${PKGDIR}/test/remote/run_remote/run_remote.go" --logtostderr --v 2 --project "${PROJECT}" --zone "${ZONE}" --ssh-env gce --delete-instances=false --cleanup=true --results-dir=my_test --service-account="${IAM_NAME}" \ No newline at end of file +go test --v=true "${PKGDIR}/test/e2e/tests" --logtostderr --project ${PROJECT} --service-account ${IAM_NAME} \ No newline at end of file diff --git a/test/run-e2e.sh b/test/run-e2e.sh index 1470209e3..1c7bf6854 100755 --- a/test/run-e2e.sh +++ b/test/run-e2e.sh @@ -5,4 +5,4 @@ set -x readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -go run "$GOPATH/src/${PKGDIR}/test/remote/run_remote/run_remote.go" --logtostderr --v 4 --zone "${ZONE}" --ssh-env gce --delete-instances=true --results-dir=my_test --run-in-prow=true +go test --v=true "${PKGDIR}/test/e2e/tests" --logtostderr --run-in-prow=true \ No newline at end of file