Use the compiled kubernetes binaries in the e2e tests in CI

Author: mauriciopoppe

test/k8s-integration/fs.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// CopyFile copies a file from src to dst
+func CopyFile(src, dst string) (err error) {
+	// get source information
+	info, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	return copyFile(src, dst, info)
+}
+
+func copyFile(src, dst string, info os.FileInfo) error {
+	// open src for reading
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		closeErr := in.Close()
+		// if we weren't returning an error
+		if err == nil {
+			err = closeErr
+		}
+	}()
+	if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
+		return err
+	}
+	// create dst file
+	// this is like f, err := os.Create(dst); os.Chmod(f.Name(), src.Mode())
+	out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode())
+	if err != nil {
+		return err
+	}
+	// make sure we close the file
+	defer func() {
+		closeErr := out.Close()
+		// if we weren't returning an error
+		if err == nil {
+			err = closeErr
+		}
+	}()
+	// actually copy
+	if _, err = io.Copy(out, in); err != nil {
+		return err
+	}
+	err = out.Sync()
+	return err
+}

test/k8s-integration/main.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"syscall"
 
@@ -662,30 +663,62 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
 
 	testArgs := fmt.Sprintf("%s %s", ginkgoArgs, testConfigArg)
 
-	kubeTestArgs := []string{
-		"--test",
-		"--ginkgo-parallel",
-		"--check-version-skew=false",
-		fmt.Sprintf("--test_args=%s", testArgs),
+	// kubetest2 flags
+
+	var runID string
+	if uid, exists := os.LookupEnv("PROW_JOB_ID"); exists && uid != "" {
+		// reuse uid for CI use cases
+		runID = uid
+	} else {
+		runID = string(uuid.NewUUID())
 	}
 
 	kubeTest2Args := []string{
 		*deploymentStrat,
+		fmt.Sprintf("--run-id=%s", runID),
 		"--test=ginkgo",
 	}
 	kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgs...)
 	if kubetestDumpDir != "" {
 		kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--artifacts=%s", kubetestDumpDir))
 	}
 	kubeTest2Args = append(kubeTest2Args, "--")
-	if len(*testVersion) != 0 && *testVersion != "master" {
-		kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=latest-%s.txt", *testVersion))
+	if len(*testVersion) != 0 {
+		if *testVersion == "master" {
+			// the kubernetes binaries should've already been built above
+			// or by the user if --localK8sDir was set, these binaries should be copied to the
+			// path sent to kubetest2 through its --artifacts path
+
+			// pkg/_artifacts is the default value that kubetests uses for --artifacts
+			kubernetesTestBinariesPath := filepath.Join(testParams.pkgDir, "_artifacts")
+			if kubetestDumpDir != "" {
+				// a custom artifacts dir was set
+				kubernetesTestBinariesPath = kubetestDumpDir
+			}
+			kubernetesTestBinariesPath = filepath.Join(kubernetesTestBinariesPath, runID)
+
+			klog.Infof("Copying kubernetes binaries to path=%s to run the tests", kubernetesTestBinariesPath)
+			err := copyKubernetesTestBinaries(testParams.k8sSourceDir, kubernetesTestBinariesPath)
+			if err != nil {
+				return fmt.Errorf("Failed to copy the kubernetes test binaries, err=%v", err)
+			}
+			kubeTest2Args = append(kubeTest2Args, "--use-built-binaries")
+		} else {
+			kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=latest-%s.txt", *testVersion))
+		}
 	}
 	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", testParams.testFocus))
 	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--skip-regex=%s", testParams.testSkip))
 	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--parallel=%d", testParams.parallel))
 	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-args=%s %s", testConfigArg, windowsArgs))
 
+	// kubetest flags
+	kubeTestArgs := []string{
+		"--test",
+		"--ginkgo-parallel",
+		"--check-version-skew=false",
+		fmt.Sprintf("--test_args=%s", testArgs),
+	}
 	if kubetestDumpDir != "" {
 		kubeTestArgs = append(kubeTestArgs, fmt.Sprintf("--dump=%s", kubetestDumpDir))
 	}
@@ -702,3 +735,30 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
 
 	return nil
 }
+
+var (
+	kubernetesTestBinaries = []string{
+		"kubectl",
+		"e2e.test",
+		"ginkgo",
+	}
+)
+
+// copyKubernetesBinariesForTest copies the common test binaries to the output directory
+func copyKubernetesTestBinaries(kuberoot string, outroot string) error {
+	const dockerizedOutput = "_output/dockerized"
+	root := filepath.Join(kuberoot, dockerizedOutput, "bin", runtime.GOOS, runtime.GOARCH)
+	for _, binary := range kubernetesTestBinaries {
+		source := filepath.Join(root, binary)
+		dest := filepath.Join(outroot, binary)
+		if _, err := os.Stat(source); err == nil {
+			klog.Infof("copying %s to %s", source, dest)
+			if err := CopyFile(source, dest); err != nil {
+				return fmt.Errorf("failed to copy %s to %s: %v", source, dest, err)
+			}
+		} else {
+			return fmt.Errorf("could not find %s: %v", source, err)
+		}
+	}
+	return nil
+}