Skip to content
This repository was archived by the owner on Apr 8, 2025. It is now read-only.

Commit b534ec8

Browse files
author
Eoghan Russell
authored
Merge pull request #164 from intel/ci
Adding end to end testing
2 parents a909934 + da104c5 commit b534ec8

File tree

15 files changed

+850
-2
lines changed

15 files changed

+850
-2
lines changed

Diff for: ‎.github/workflows/e2e.yml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: E2E
2+
on: [push, pull_request]
3+
jobs:
4+
5+
e2e:
6+
name: E2E
7+
runs-on: hugepage-runner
8+
steps:
9+
- name: Set up Go
10+
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
11+
with:
12+
go-version: 1.20.1
13+
- uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
14+
- name: install_go_kubectl_kind
15+
run: source ./ci/ci.sh && install_go_kubectl_kind
16+
- name: create_kind_cluster
17+
run: source ./ci/ci.sh && create_kind_cluster
18+
- name: deploy_multus
19+
run: source ./ci/ci.sh && deploy_multus
20+
- name: vpp_e2e_test
21+
run: source ./ci/ci.sh && vpp_e2e_test
22+
- name: build_ovs_container
23+
run: source ./ci/ci.sh && build_ovs_container
24+
- name: build_test-pmd_container
25+
run: source ./ci/ci.sh && build_testpmd_container
26+
- name: ovs_e2e_test
27+
run: source ./ci/ci.sh && ovs_e2e_test
28+
29+

Diff for: ‎.github/workflows/scorecard.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,4 @@ jobs:
6969
- name: "Upload to code-scanning"
7070
uses: github/codeql-action/upload-sarif@0ba4244466797eb048eb91a6cd43d5c03ca8bd05 # v2.21.2
7171
with:
72-
sarif_file: results.sarif
72+
sarif_file: results.sarif

Diff for: ‎ci/ci.sh

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/bin/bash
2+
set -e
3+
CI_DIR="/runner/_work/userspace-cni-network-plugin/userspace-cni-network-plugin/ci/"
4+
5+
install_go_kubectl_kind(){
6+
wget -qO- https://golang.org/dl/go1.20.1.linux-amd64.tar.gz |tar -C /home/runner -xz
7+
export PATH="${PATH}:/home/runner/go/bin"
8+
echo "export PATH=\"${PATH}:/home/runner/go/bin/:home/runner/.local/bin/\"" >>~/.bashrc
9+
go install sigs.k8s.io/[email protected]
10+
11+
wget -q https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl -O /home/runner/go/bin/kubectl
12+
chmod +x /home/runner/go/bin/kubectl
13+
}
14+
15+
16+
create_kind_cluster(){
17+
kind create cluster
18+
kubectl get all --all-namespaces
19+
20+
#docker run -itd --device=/dev/hugepages:/dev/hugepages --privileged -v "$(pwd)/docker/vpp-centos-userspace-cni/:/etc/vpp/" --name vpp ligato/vpp-base
21+
sleep 10
22+
cd /runner/_work/userspace-cni-network-plugin/userspace-cni-network-plugin/
23+
24+
docker build . -f ./docker/userspacecni/Dockerfile -t userspacecni:latest
25+
kind load docker-image userspacecni
26+
docker exec -i kind-control-plane bash -c "mkdir -p '/runner/_work/userspace-cni-network-plugin/'"
27+
docker cp "/runner/_work/userspace-cni-network-plugin/userspace-cni-network-plugin/" "kind-control-plane:/runner/_work/userspace-cni-network-plugin/"
28+
}
29+
30+
deploy_multus(){
31+
## Multus main branch has major bugs so we fix version
32+
cd $CI_DIR
33+
MULTUS_VERSION="v4.0.2"
34+
wget https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/$MULTUS_VERSION/deployments/multus-daemonset.yml
35+
sed -i "s/snapshot-thick/v4.0.2/g" multus-daemonset.yml
36+
kubectl apply -f ./multus-daemonset.yml
37+
}
38+
39+
vpp_e2e_test(){
40+
cd $CI_DIR/vpp_test_setup/
41+
echo "Setting up vpp host"
42+
./vpp_host.sh
43+
sleep 20
44+
echo "Setting up vpp pods"
45+
./vpp_pod.sh
46+
47+
sleep 20
48+
kubectl get all -A
49+
50+
kubectl exec -n vpp vpp-app1-kind-control-plane -- ./vpp_pod_setup_memif.sh
51+
kubectl exec -n vpp vpp-app2-kind-control-plane -- ./vpp_pod_setup_memif.sh
52+
kubectl exec -n vpp vpp-app1-kind-control-plane -- vppctl "sh int address"
53+
kubectl exec -n vpp vpp-app2-kind-control-plane -- vppctl "sh int address"
54+
55+
kubectl exec -n vpp vpp-app1-kind-control-plane -- vppctl "ping 192.168.1.4"
56+
if kubectl exec -n vpp vpp-app1-kind-control-plane -- vppctl "ping 192.168.1.4" |grep -q "bytes"; then
57+
echo "VPP ping test pass"
58+
else
59+
echo VPP ping test failed
60+
exit 1
61+
fi
62+
}
63+
64+
build_ovs_container(){
65+
cd $CI_DIR/ovs_test_setup
66+
docker build . -t ovs
67+
kind load docker-image ovs
68+
# set alias for ovs-vsctl and ovs-ofctl in kind container, would be better if we could just install its bin
69+
docker exec -i kind-control-plane bash -c "echo '#!/bin/bash' > /usr/bin/ovs-vsctl"
70+
docker exec -i kind-control-plane bash -c "echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /usr/bin/ovs-vsctl"
71+
docker exec -i kind-control-plane bash -c "echo 'kubectl exec -n ovs ovs-kind-control-plane -- ovs-vsctl \"\$@\"' >> /usr/bin/ovs-vsctl"
72+
docker exec -i kind-control-plane bash -c "chmod +x /usr/bin/ovs-vsctl"
73+
74+
docker exec -i kind-control-plane bash -c "echo '#!/bin/bash' > /usr/bin/ovs-ofctl"
75+
docker exec -i kind-control-plane bash -c "echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /usr/bin/ovs-ofctl"
76+
docker exec -i kind-control-plane bash -c "echo 'kubectl exec -n ovs ovs-kind-control-plane -- ovs-ofctl \"\$@\"' >> /usr/bin/ovs-ofctl"
77+
docker exec -i kind-control-plane bash -c "chmod +x /usr/bin/ovs-ofctl"
78+
}
79+
80+
build_testpmd_container(){
81+
cd $CI_DIR/ovs_test_setup/testpmd_image
82+
docker build -t testpmd .
83+
kind load docker-image testpmd
84+
}
85+
86+
ovs_e2e_test(){
87+
cd $CI_DIR/ovs_test_setup
88+
./ovs_host.sh
89+
sleep 20
90+
91+
# workaround, cant create in dockerfile
92+
kubectl exec -n ovs pod/ovs-kind-control-plane -- bash -c "mkdir -p /dev/net/"
93+
kubectl exec -n ovs pod/ovs-kind-control-plane -- bash -c "mknod /dev/net/tun c 10 200"
94+
kubectl exec -n ovs pod/ovs-kind-control-plane -- bash -c 'ovs-vsctl set Open_vSwitch . "other_config:dpdk-init=true"'
95+
96+
./testpmd_pod.sh
97+
98+
sleep 30
99+
kubectl get all -A
100+
kubectl logs -n ovs pod/ovs-kind-control-plane
101+
kubectl describe -n ovs pod/ovs-app1-kind-control-plane
102+
kubectl logs -n ovs pod/ovs-app1-kind-control-plane |tail -11
103+
kubectl logs -n ovs pod/ovs-app2-kind-control-plane |tail -11
104+
pps="$(kubectl logs -n ovs pod/ovs-app2-kind-control-plane |tail -11 | grep 'RX-packets'|sed 's/ * / /g' |cut -d ' ' -f 3)"
105+
echo "RX Packets: $pps"
106+
107+
if [ "$pps" -eq "0" ] || [ -z "${pps}" ]; then
108+
echo "Test Failed: no traffic";
109+
exit 1;
110+
else
111+
echo "OVS Test Pass";
112+
fi
113+
}
114+
115+
116+
run_all(){
117+
# theese steps are triggered by the ci by sourcing this script and running the following separately
118+
# it gives much better logging breakdown on github
119+
# the run_all function is only used for manual deployment
120+
install_go_kubectl_kind
121+
create_kind_cluster
122+
deploy_multus
123+
vpp_e2e_test
124+
build_ovs_container
125+
build_testpmd_container
126+
ovs_e2e_test
127+
}
128+
#run_all

Diff for: ‎ci/ovs_test_setup/Dockerfile

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
FROM ubuntu:22.04
2+
RUN apt-get -q update && apt-get install -y openvswitch-switch-dpdk
3+
RUN update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk
4+
##RUN sed -i "/rmmod bridge/d" /usr/share/openvswitch/scripts/ovs-kmod-ctl
5+
RUN apt install -y linux-headers-$(uname -r)
6+
CMD /usr/share/openvswitch/scripts/ovs-ctl start && sleep inf

Diff for: ‎ci/ovs_test_setup/ovs_host.sh

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
kubectl delete ns ovs
3+
kubectl create ns ovs
4+
5+
worker="kind-control-plane"
6+
cat << EOF | kubectl apply -f -
7+
---
8+
apiVersion: v1
9+
kind: Pod
10+
metadata:
11+
name: ovs-$worker
12+
labels:
13+
name: ovs
14+
namespace: ovs
15+
spec:
16+
hostNetwork: false
17+
nodeSelector:
18+
kubernetes.io/hostname: $worker
19+
hostname: ovs-$worker
20+
subdomain: ovs
21+
containers:
22+
- image: ovs
23+
imagePullPolicy: IfNotPresent
24+
name: ovs-$worker
25+
volumeMounts:
26+
- name: vpp-api
27+
mountPath: /run/openvswitch
28+
- name: modules
29+
mountPath: /lib/modules
30+
- name: vpp-run
31+
mountPath: /var/run/openvswitch
32+
- name: hugepage
33+
mountPath: /hugepages
34+
resources:
35+
requests:
36+
hugepages-2Mi: 1Gi
37+
memory: "1Gi"
38+
cpu: "10"
39+
limits:
40+
hugepages-2Mi: 1Gi
41+
memory: "1Gi"
42+
cpu: "10"
43+
securityContext:
44+
capabilities:
45+
add: ["NET_ADMIN", "SYS_TIME"]
46+
restartPolicy: Always
47+
volumes:
48+
- name: vpp-run
49+
hostPath:
50+
path: /var/run/openvswitch/
51+
- name: modules
52+
hostPath:
53+
path: /lib/modules
54+
- name: vpp-api
55+
hostPath:
56+
path: /run/openvswitch/
57+
- name: userspace-api
58+
hostPath:
59+
path: /var/lib/cni/usrspcni/
60+
- name: hugepage
61+
emptyDir:
62+
medium: HugePages
63+
EOF
64+

Diff for: ‎ci/ovs_test_setup/ovs_network_attachment_definition.yaml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
apiVersion: "k8s.cni.cncf.io/v1"
2+
kind: NetworkAttachmentDefinition
3+
metadata:
4+
name: userspace-ovs-net-1
5+
namespace: ovs
6+
spec:
7+
config: '{
8+
"cniVersion": "0.3.1",
9+
"type": "userspace",
10+
"name": "userspace-ovs-net-1",
11+
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
12+
"logFile": "/var/log/userspace-ovs-net-1-cni.log",
13+
"logLevel": "debug",
14+
"host": {
15+
"engine": "ovs-dpdk",
16+
"iftype": "vhostuser",
17+
"netType": "bridge",
18+
"vhost": {
19+
"mode": "client"
20+
},
21+
"bridge": {
22+
"bridgeName": "br-4"
23+
}
24+
},
25+
"container": {
26+
"engine": "ovs-dpdk",
27+
"iftype": "vhostuser",
28+
"netType": "interface",
29+
"vhost": {
30+
"mode": "server"
31+
}
32+
},
33+
"ipam": {
34+
"type": "host-local",
35+
"subnet": "10.56.217.0/24",
36+
"rangeStart": "10.56.217.131",
37+
"rangeEnd": "10.56.217.190",
38+
"routes": [
39+
{
40+
"dst": "0.0.0.0/0"
41+
}
42+
],
43+
"gateway": "10.56.217.1"
44+
}
45+
}'

Diff for: ‎ci/ovs_test_setup/testpmd_image/Dockerfile

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM ubuntu:22.04
2+
RUN apt-get -q update && apt-get install -y git vim meson python3-pyelftools libnuma-dev
3+
RUN git clone https://github.com/DPDK/dpdk.git
4+
WORKDIR /dpdk/
5+
RUN meson build
6+
RUN cd build && ninja
7+
WORKDIR /dpdk/build/app
8+
COPY ./testpmd.sh testpmd.sh
9+
CMD ./testpmd.sh

Diff for: ‎ci/ovs_test_setup/testpmd_image/testpmd.sh

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
cd /dpdk || exit 1
4+
#set container id as env
5+
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/x86_64-linux-gnu/
6+
while ! [ -s /etc/podinfo/annotations ]; do
7+
echo "annotations not found yet"
8+
sleep 1 # throttle the check
9+
done
10+
cat /etc/podinfo/annotations
11+
sleep 10
12+
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
13+
##containerid=$(grep containerID /etc/podinfo/annotations |cut -d = -f 2 | sed 's/"//g')
14+
containerid=$(grep containerId /etc/podinfo/annotations |cut -d '"' -f 5 |sed 's/\\//')
15+
echo "${containerid:0:12}"
16+
#./build/app/dpdk-testpmd -l 10,14,12 --vdev net_vhost0,iface=/var/lib/cni/usrspcni/${containerid:0:12}-net1 -- -i
17+
cpu=$(cat /sys/fs/cgroup/cpuset.cpus)
18+
#--vdev "net_virtio_user0,path=/run/openvswitch/host/host-net1,server=1,queue_size=2048" --in-memory --single-file-segments --no-pci
19+
20+
if grep -q app1 /etc/podinfo/labels; then
21+
fordwardmode="txonly"
22+
cpu="2,3,4,5"
23+
else
24+
fordwardmode="rxonly"
25+
cpu="6,7,8,9"
26+
fi
27+
28+
#--stats-period 1 is needed to avoid testpmd exiting
29+
commands="./build/app/dpdk-testpmd -l $cpu --vdev net_virtio_user0,path=/var/lib/cni/usrspcni/${containerid:0:12}-net1,server=1,queue_size=2048 --in-memory --single-file-segments -- --tx-ip 192.168.1.1,192.168.1.2 --tx-udp=4000,4000 --forward-mode=$fordwardmode --stats-period 1" #-i
30+
echo "$commands"
31+
$commands

0 commit comments

Comments
 (0)