Skip to content

Commit f5a4ff9

Browse files
mauriciopoppemauriciopoppe
committed
Run e2e tests in GCE clusters with additional test-args
Initial commit for windows remote hostexec impl
1 parent dd99689 commit f5a4ff9

File tree

7 files changed

+207
-7
lines changed

7 files changed

+207
-7
lines changed

‎hack/e2e.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ func main() {
7777
if err != nil {
7878
log.Fatalf("err: %v", err)
7979
}
80-
log.Printf("Calling kubetest %v...", strings.Join(f.args, " "))
80+
log.Printf("Calling [kubetest %v]", strings.Join(f.args, " "))
8181
if err = t.wait(k, f.args...); err != nil {
8282
log.Fatalf("err: %v", err)
8383
}

‎hack/e2e.sh

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -293,13 +293,13 @@ if [ "${1:-}" == "--" ]; then
293293
fi
294294

295295
if [ "$PROVIDER" == "gke" ]; then
296-
# --up
297-
# --down
298296
kubetest2_args+=(
299297
--test exec
300298
-v 1
301299
--cluster-name "$CLUSTER"
302300
--network "$CLUSTER"
301+
--up
302+
--down
303303
--gcp-service-account "$GOOGLE_APPLICATION_CREDENTIALS"
304304
--environment "$GKE_ENVIRONMENT"
305305
)
@@ -324,6 +324,5 @@ fi
324324
# legacy path
325325
go run $ROOT/hack/e2e.go -- "${kubetest_args[@]}" \
326326
--deployment "$DEPLOYMENT" \
327-
--test-cmd bash \
328-
--test-cmd-args="$ROOT/hack/run-e2e.sh" \
329-
"$@"
327+
--test-cmd "$ROOT/hack/run-e2e.sh" \
328+
--test-cmd-args="$@"

‎hack/run-e2e.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ fi
4545
KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-} # e.g. local, gce
4646
KUBERNETES_CONFORMANCE_TEST=${KUBERNETES_CONFORMANCE_TEST:-}
4747
KUBERNETES_CONFORMANCE_PROVIDER=${KUBERNETES_CONFORMANCE_PROVIDER:-}
48+
KUBERNETES_NODE_PLATFORM=${KUBERNETES_NODE_PLATFORM:-} # e.g. windows
4849
KUBE_GCE_ZONE=${KUBE_GCE_ZONE:-} # Available when provider is gce/gke
4950
PROJECT=${PROJECT:-} # Available when provider is gce/gke
5051
KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
@@ -132,6 +133,22 @@ fi
132133
export PROVISIONER_IMAGE_NAME
133134
export PROVISIONER_IMAGE_PULL_POLICY
134135

136+
current_platform=linux
137+
taint_platform=windows
138+
if [ "${KUBERNETES_NODE_PLATFORM}" == "windows" ]; then
139+
current_platform=windows
140+
taint_platform=linux
141+
fi
142+
echo "Running tests for platform=${current_platform}, removing taints from all nodes and tainting platform=${taint_platform} nodes"
143+
kubectl get nodes -o name | \
144+
while IFS= read -r node; do
145+
kubectl taint node $node node.kubernetes.io/os:NoSchedule- || true
146+
done
147+
kubectl get nodes -l kubernetes.io/os=${taint_platform} -o name | \
148+
while IFS= read -r node; do
149+
kubectl taint node $node node.kubernetes.io/os:NoSchedule || true
150+
done
151+
135152
TEST_ARGS=(
136153
test
137154
-timeout=60m

‎hack/windows_remote_exec.sh

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
# Copyright 2021 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -ex
18+
19+
function get_windows_node() {
20+
echo $(kubectl get nodes -l kubernetes.io/os=windows -o jsonpath='{.items[*].metadata.name}')
21+
}
22+
23+
function main() {
24+
echo "Compiling the test program"
25+
local output="_output/windows/amd64/vhd.exe"
26+
GO111MODULE=off CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -ldflags '-extldflags "-static"' -o $output ./test/windows
27+
28+
echo "Checking that the target windows node exists"
29+
local windows_node=$(get_windows_node)
30+
gcloud compute instances describe $windows_node > /dev/null
31+
32+
echo "Executing the program remotely"
33+
local current_account=$(gcloud config list account --format "value(core.account)" | sed -r 's/@\S+//g')
34+
gcloud compute scp $output $windows_node:"C:\\Users\\${current_account}"
35+
gcloud compute ssh $windows_node --command="powershell -c .\vhd.exe"
36+
}
37+
38+
main $@

‎test/e2e/e2e_test.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ const (
8080
)
8181

8282
var (
83-
// provisioner image used for e2e tests
83+
// default provisioner image used for e2e tests
8484
provisionerImageName = "quay.io/external_storage/local-volume-provisioner:latest"
8585
provisionerImagePullPolicy v1.PullPolicy = "Never"
8686
// storage class volume binding modes
@@ -161,6 +161,9 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
161161

162162
BeforeEach(func() {
163163
// Get all the schedulable nodes
164+
// NOTE: After the creation of the e2e cluster there's a script
165+
// that taints the nodes that don't belong to the current platform
166+
// being tested e.g. taint the Linux nodes if the tests are for Windows
164167
nodes, err := e2enode.GetReadySchedulableNodes(f.ClientSet)
165168
framework.ExpectNoError(err)
166169
Expect(len(nodes.Items)).NotTo(BeZero(), "No available nodes for scheduling")

‎test/windows/host_exec.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
"os"
23+
"os/exec"
24+
25+
v1 "k8s.io/api/core/v1"
26+
e2estorageutils "k8s.io/kubernetes/test/e2e/storage/utils"
27+
)
28+
29+
type HostExecutor struct {
30+
}
31+
32+
var _ e2estorageutils.HostExec = &HostExecutor{}
33+
34+
// NewHostExec returns a HostExec
35+
func NewHostExec() *HostExecutor {
36+
return &HostExecutor{}
37+
}
38+
39+
// Execute executes the command on the given node. If there is no error
40+
// performing the remote command execution, the stdout, stderr and exit code
41+
// are returned.
42+
func (h *HostExecutor) Execute(command string, node *v1.Node) (e2estorageutils.Result, error) {
43+
powershellCommand := fmt.Sprintf(`--command=powershell -c %s`, command)
44+
args := []string{
45+
"compute",
46+
"ssh",
47+
node.Name,
48+
powershellCommand,
49+
}
50+
var outBuffer, errBuffer bytes.Buffer
51+
cmd := exec.Command("gcloud", args...)
52+
cmd.Env = os.Environ()
53+
cmd.Stdout = &outBuffer
54+
cmd.Stderr = &errBuffer
55+
56+
err := cmd.Run()
57+
result := e2estorageutils.Result{
58+
Host: node.Name,
59+
Cmd: cmd.String(),
60+
Stdout: outBuffer.String(),
61+
Stderr: errBuffer.String(),
62+
}
63+
if err != nil {
64+
if exitError, ok := err.(*exec.ExitError); ok {
65+
result.Code = exitError.ExitCode()
66+
}
67+
e2estorageutils.LogResult(result)
68+
return result, err
69+
}
70+
71+
return result, nil
72+
}
73+
74+
// IssueCommandWithResult issues command on the given node and returns stdout as
75+
// result. It returns error if there are some issues executing the command or
76+
// the command exits non-zero.
77+
func (h *HostExecutor) IssueCommandWithResult(cmd string, node *v1.Node) (string, error) {
78+
result, err := h.Execute(cmd, node)
79+
if err != nil {
80+
e2estorageutils.LogResult(result)
81+
}
82+
return result.Stdout, err
83+
}
84+
85+
// IssueCommand works like IssueCommandWithResult, but discards result.
86+
func (h *HostExecutor) IssueCommand(cmd string, node *v1.Node) error {
87+
_, err := h.IssueCommandWithResult(cmd, node)
88+
return err
89+
}
90+
91+
// Cleanup cleanup resources it created during test.
92+
// Note that in most cases it is not necessary to call this because we create
93+
// pods under test namespace which will be destroyed in teardown phase.
94+
func (h *HostExecutor) Cleanup() {
95+
}

‎test/windows/main.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"flag"
21+
22+
v1 "k8s.io/api/core/v1"
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
25+
"k8s.io/klog/v2"
26+
e2estorageutils "k8s.io/kubernetes/test/e2e/storage/utils"
27+
)
28+
29+
func main() {
30+
klog.InitFlags(nil)
31+
defer klog.Flush()
32+
33+
flag.Parse()
34+
35+
klog.Infof("Remotely executing a program!")
36+
37+
node := &v1.Node{
38+
ObjectMeta: metav1.ObjectMeta{
39+
Name: "e2e-test-mauriciopoppe-windows-node-group-l64t",
40+
},
41+
}
42+
hostExec := NewHostExec()
43+
result, err := hostExec.Execute("echo $env:USERPROFILE", node)
44+
if err != nil {
45+
panic(err)
46+
}
47+
e2estorageutils.LogResult(result)
48+
}

0 commit comments

Comments
 (0)