diff --git a/.github/workflows/trivy-testpmd.yml b/.github/workflows/trivy-testpmd.yml new file mode 100644 index 00000000..22f3e980 --- /dev/null +++ b/.github/workflows/trivy-testpmd.yml @@ -0,0 +1,47 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: trivy-testpmd + +on: + push: + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + schedule: + - cron: '40 20 * * 4' + +permissions: + contents: read + +jobs: + build: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + + - name: Build testpmd image + run: docker build -t testpmd:latest -f ./docker/testpmd/Dockerfile ./docker/testpmd/ + + - name: Testpmd Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@f78e9ecf42a1271402d4f484518b9313235990e1 + with: + image-ref: 'testpmd:latest' + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'testpmd-trivy-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'testpmd-trivy-results.sarif' diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index de6f3264..4dee85d2 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -47,4 +47,3 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: 'trivy-results.sarif' - diff --git a/Makefile b/Makefile index 40b22963..74dac8be 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,7 @@ deploy: undeploy: kubectl delete -f ./kubernetes/userspace-daemonset.yml + +testpmd: + @$(IMAGE_BUILDER) build -t $(IMAGE_REGISTRY)testpmd:latest -f ./docker/testpmd/Dockerfile ./docker/testpmd/ + @$(IMAGE_BUILDER) push $(IMAGE_REGISTRY)testpmd:latest diff --git a/README.md b/README.md index a31492d0..7af6bedc 100644 --- a/README.md +++ b/README.md @@ -102,53 +102,28 @@ for more information. # Build & Clean -This plugin is recommended to be built with Go 1.11.10 and either OVS-DPDK 2.9.0-3 -or VPP 19.04. Other versions of Go, OVS-DPDK and VPP are theoretically +This plugin is recommended to be built with Go 1.20.1 and either OVS-DPDK 2.9.0-3 +or VPP 23.02. Other versions of Go, OVS-DPDK and VPP are theoretically supported, but MIGHT cause unknown issue. -There are a few environmental variables used in building and testing this plugin. -Here is an example: -``` - cat ~/.bashrc - : - export GOPATH=~/go - export CNI_PATH=$GOPATH/src/github.com/containernetworking/plugins/bin -``` - +The Userspace CNI requires several files from VPP in-order to build. +For this reason we build userspacecni in a container. -The Userspace CNI requires several files from VPP in-order to build. If VPP -should be installed but is not installed, see [Installing VPP](#installing-vpp) -section below for instructions. If Userspace CNI is being built on a build -server or is using OVS-DPDK (i.e. - don't want VPP installed), then follow the -instructions below under -[Building VPP CNI Library with OVS](#building-vpp-cni-library-with-ovs). - -To get and build the Userspace CNI plugin: -``` - cd $GOPATH/src/ - go get github.com/intel/userspace-cni-network-plugin - cd github.com/intel/userspace-cni-network-plugin - make -``` +userspacecni is built in a container and then transferred to the host. +By default Docker is used to build the image -Once the binary is built, it needs to be copied to the CNI directory: +To build the docker image: ``` - cp userspace/userspace $CNI_PATH/. + git clone https://github.com/intel/userspace-cni-network-plugin.git + cd userspace-cni-network-plugin + make build ``` -To remove the binary and temporary files generated while building the source -code, perform a make clean: +To copy the userspacecni binary to the host directory /opt/cni/bin/ ``` - make clean + make deploy ``` - -## Update dependencies -This project is currently using [go modules](https://github.com/golang/go/wiki/Modules) -to manage dependencies. Please refer to official documentation to learn more -about **go modules** behavior and typical [workflow](https://github.com/golang/go/wiki/Modules#daily-workflow). - - # Network Configuration Reference * `type` (string, required): "userspace" @@ -388,42 +363,9 @@ Instance. So to build the VPP CNI, VPP must be installed (or the proper json files must be in */usr/share/vpp/api/*). -## Building VPP CNI Library with OVS -The Userspace CNI plugin builds the VPP CNI Library from the cnivpp -sub-folder. In order to run with the VPP CNI Library, VPP must be installed -on the system. If VPP should be installed but is not installed, see the -[Installing VPP](#installing-vpp) section below. - -If the desire is to run the OVS CNI Library with OVS (i.e. - don't want -VPP installed), several files from a typical VPP install need to be on -the system to build. To install just these files and NOT VPP, run: -``` - cd $GOPATH/src/ - go get github.com/Billy99/user-space-net-plugin - cd github.com/Billy99/user-space-net-plugin - make install -``` -This will install only the 5 or 6 files needed to build the VPP CNI Library. -To remove these files, run: -``` - make clean -``` -*make install* requires several packages to execute, primarily *wget*, -*cpio* and *rpm2cpio* on CentOS, and *binutils* on Ubuntu. If these packages are -not installed on your system, the following can be run to install the required -packages: -``` - make install-dep -``` -**NOTE:** *make install* has been made to work for CentOS and Ubuntu based -systems. Other platforms will be made to work long term. If there is an -immediate need for other platforms, please open an issue to expedite the -feature (https://github.com/intel/userspace-cni-network-plugin/issues). - - ## Installing VPP There are several ways to install VPP. This code is based on a fixed release -VPP (VPP 19.04 initially), so it is best to install a released version (even +VPP (VPP 23.02), so it is best to install a released version (even though it is possible to build your own). @@ -445,135 +387,52 @@ hugepages to 512, use: * **SELinux:** VPP works with SELinux enabled, but when running with containers, work still needs to be done. Set SELinux to permissive. +### VPP install guide +https://s3-docs.fd.io/vpp/23.02/gettingstarted/installing/ -### Install on CentOS -To install VPP on CentOS from https://packagecloud.io/fdio/ repository: -``` -curl -s https://packagecloud.io/install/repositories/fdio/1904/script.rpm.sh | sudo bash -``` -To resolve dependency issues, install the following packages: -``` -yum install -y epel-release mbedtls python36 -``` -Installing RPMs: -``` -yum install -y vpp vpp-lib vpp-plugins vpp-devel vpp-api-python vpp-api-lua vpp-selinux-policy -``` - -To start and enable VPP: -``` -sudo systemctl start vpp -sudo systemctl enable vpp -``` - -### Install on Ubuntu -**OLD - Needs to be updated!** - -To install on Ubuntu 16.04 (Xenial) as an example to demonstrate how to install VPP from pre-build packages: -``` -export UBUNTU="xenial" -export RELEASE=".stable.1904" -sudo rm /etc/apt/sources.list.d/99fd.io.list -echo "deb [trusted=yes] https://nexus.fd.io/content/repositories/fd.io$RELEASE.ubuntu.$UBUNTU.main/ ./" | sudo tee -a /etc/apt/sources.list.d/99fd.io.list -sudo apt-get update -sudo apt-get install vpp vpp-lib -``` - - -### Install on Other Distros -For installing VPP on other distros, see: -https://wiki.fd.io/view/VPP/Installing_VPP_binaries_from_packages +### VPP source code +https://github.com/FDio/vpp/tree/v23.02 +### OVS install guide +https://docs.openvswitch.org/en/latest/intro/install/ +### OVS source code +https://github.com/openvswitch/ovs/tree/master # Testing -## Testing with VPP Docker Image and CNI - -There are a few environmental variables used in this test. Here is an example: -``` - cat ~/.bashrc - : - export GOPATH=~/go - export CNI_PATH=$GOPATH/src/github.com/containernetworking/plugins/bin +## Testing VPP with VPP pod as application +Its assumed that VPP has been set up on the host correctly by the user. +On the server that the vpp pods get deployed (not the k8 controller) create the directories `/var/run/vpp/app1` +and `/var/run/vpp/app2` for this example to work ``` +kubectl create ns vpp -In order to test, a container with VPP 19.04 and usrsp-app has been created: -``` - docker pull bmcfall/vpp-centos-userspace-cni:latest -``` -More details on the Docker Image, how to build from scratch and other -information, see -[README.md](https://github.com/intel/userspace-cni-network-plugin/blob/master/docker/vpp-centos-userspace-cni/README.md) -in the '*./docker/vpp-centos-userspace-cni/*' subfolder. +kubectl create -f ./examples/vpp-memif-ping/userspace-vpp-netAttach.yaml -Setup your configuration file in your CNI directory. An example is -*/etc/cni/net.d/*. +kubectl create -n vpp configmap vpp-app-startup-config --from-file=./examples/vpp-memif-ping/startup.conf -**NOTE:** The *userspace* netconf definition is still a work in progress. So -the example below is just an example, see *pkg/types/types.go* for latest definitions. +kubectl create -n vpp configmap vpp-pod-setup-memif --from-file=./examples/vpp-memif-ping/vpp-pod-setup-memif.sh -Example of how to setup a configuration for a VPP memif interface between the -host and container: -``` -sudo vi /etc/cni/net.d/90-userspace.conf -{ - "cniVersion": "0.3.1", - "type": "userspace", - "name": "memif-network", - "host": { - "engine": "vpp", - "iftype": "memif", - "netType": "bridge", - "memif": { - "role": "master", - "mode": "ethernet" - }, - "bridge": { - "bridgeName": "4" - } - }, - "container": { - "engine": "vpp", - "iftype": "memif", - "netType": "interface", - "memif": { - "role": "slave", - "mode": "ethernet" - } - }, - "ipam": { - "type": "host-local", - "subnet": "192.168.210.0/24", - "routes": [ - { "dst": "0.0.0.0/0" } - ] - } -} -``` +kubectl create -f ./examples/vpp-memif-ping/vpp-app-pod-1.yaml +kubectl create -f ./examples/vpp-memif-ping/vpp-app-pod-2.yaml -To test, currently using a local script (copied from CNI scripts): -https://github.com/containernetworking/cni/blob/master/scripts/docker-run.sh). -To run script: -``` - cd $GOPATH/src/github.com/intel/userspace-cni-network-plugin/ - sudo CNI_PATH=$CNI_PATH GOPATH=$GOPATH ./scripts/usrsp-docker-run.sh -it --privileged vpp-centos-userspace-cni -``` +kubectl exec -itn vpp vpp-app1 -- ./vpp-pod-setup-memif.sh +kubectl exec -itn vpp vpp-app2 -- ./vpp-pod-setup-memif.sh -**NOTE:** The *usrsp-docker-run.sh* script mounts some volumes in the container. Change as needed: -* *-v /var/lib/cni/usrspcni/shared:/var/lib/cni/usrspcni/shared:rw* - * Socket files (memif or vhost-user) are passed to the container through a subdirectory of this base directory.. -* *-v /var/lib/cni/usrspcni/$contid:/var/lib/cni/usrspcni/data:rw* - * Current implementation is to write the remote configuration into a file and share the directory -with the container, which is the volume mapping. Directory is currently hard coded. -* *--device=/dev/hugepages:/dev/hugepages* - * VPP requires hugepages, so need to map hugepages into container. +# ping pod 2 from pod 1 (through vpp) +kubectl exec -itn vpp vpp-app1 -- vppctl "ping 192.168.1.4" +# sample output +116 bytes from 192.168.1.4: icmp_seq=1 ttl=64 time=83.4641 ms +116 bytes from 192.168.1.4: icmp_seq=2 ttl=64 time=72.0057 ms +116 bytes from 192.168.1.4: icmp_seq=3 ttl=64 time=67.9981 ms +116 bytes from 192.168.1.4: icmp_seq=4 ttl=64 time=75.9974 ms +116 bytes from 192.168.1.4: icmp_seq=5 ttl=64 time=72.0021 ms -In the container, you should see the usrsp-app ouput the message sequence of -its communication with local VPP (VPP in the container) and some database -dumps interleaved. +``` +### Optional debug steps for VPP ### Verify Host To verify the local config (on host) in another window: @@ -600,7 +459,9 @@ After the container is started, on the host there should be an additional memif interface created and added to a new L2 bridge, all created by the Userspace CNI. -**After Container Started:** + + +**After vpp pod application Started:** ``` vppctl show interface Name Idx State Counter Count @@ -693,24 +554,9 @@ vppctl ping 192.168.210.46 Statistics: 5 sent, 4 received, 20% packet loss ``` -### Debug -The *vpp-centos-userspace-cni* container runs a script at startup (in Dockefile -CMD command) which starts VPP and then runs *usrsp-app*. Assuming the same notes -above, to see what is happening in the container, cause -*vpp-centos-userspace-cni* container to start in bash and skip the script, then -run VPP and *usrsp-app* manually: -``` - cd $GOPATH/src/github.com/containernetworking/cni/scripts - sudo CNI_PATH=$CNI_PATH GOPATH=$GOPATH ./scripts/usrsp-docker-run.sh -it --privileged bmcfall/vpp-centos-userspace-cni:0.2.0 bash - - /* Within Container: */ - vpp -c /etc/vpp/startup.conf & - usrsp-app -``` - - ## Testing with DPDK Testpmd Application + To follow this example you should have a system with kubernetes available and configured to support native 1 GB hugepages. You should also have multus-cni and userspace-cni-network-plugin up and running. See `examples/crd-userspace-net-ovs-no-ipam.yaml` for @@ -719,304 +565,37 @@ check that you have bridge named `br0` in your OVS with `ovs-vsctl show` and if not, create it with `ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev`. -### 1. Build the image to be used - -Build container image from - -```Dockerfile -FROM ubuntu:bionic - -RUN apt-get update && apt-get install -y dpdk; - -ENTRYPOINT ["bash"] +Build testpmd container image ``` - -Dockerfile and tag it as `ubuntu-dpdk`: - -```bash -docker build . -t ubuntu-dpdk +make testpmd ``` -### 2. Create pod with multiple vhostuser interfaces +Modify the nodeSelector `kubernetes.io/hostname:` in both `./examples/ovs-vhost/testpmd-pod-1.ymal` and `./examples/ovs-vhost/testpmd-pod-2.ymal` -Copy `get-prefix.sh` script from userspace-cni-network-plugin repo to -`/var/lib/cni/vhostuser/`. See `examples/pod-multi-vhost.yaml`and start the -pod: - -```bash -kubectl create -f examples/pod-multi-vhost.yaml -``` - -### 3. Open terminal to pod and start testpmd - -Open terminal to the created pod once it is running: - -```bash -kubectl exec -it multi-vhost-example bash -``` - -Launch testpmd and automatically start forwarding packets after sending first -burst: - -```bash -# Get container ID -export ID=$(/vhu/get-prefix.sh) - -# Run testpmd with ports created by vhostplugin -# Note: change coremask to suit your system -testpmd \ - -d librte_pmd_virtio.so.17.11 \ - -m 1024 \ - -c 0xC \ - --file-prefix=testpmd_ \ - --vdev=net_virtio_user0,path=/vhu/${ID}/${ID:0:12}-net1 \ - --vdev=net_virtio_user1,path=/vhu/${ID}/${ID:0:12}-net2 \ - --no-pci \ - -- \ - --no-lsc-interrupt \ - --auto-start \ - --tx-first \ - --stats-period 1 \ - --disable-hw-vlan; ``` - -If packets are not going through, you may need to configure direct flows to your -switch between the used ports. For example, with OVS as the switch, this is done -by getting the port numbers with `ovs-ofctl dump-ports br0` and configuring -flow, for example, from port 1 to port 2 with -`ovs-ofctl add-flow br0 in_port=1,action=output:2` and vice versa. - -## Testing with DPDK L3FWD Application - -To follow this example you should have a system with kubernetes available and configured to support native 1 GB hugepages. You should also have multus-cni and userspace-cni-network-plugin up and running. See `examples/crd-userspace-net-ovs-no-ipam.yaml` for example config to use with multus. If using OVS, check that you have bridge named `br0` in your OVS with `ovs-vsctl show` and if not, create it with `ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev`. - -### 1. Build the images to be used - -Create the l3fwd Dockerfile: -```bash -$ cat < l3fwd-dockerfile -FROM ubuntu:bionic - -RUN apt-get update && apt-get install -y --no-install-recommends \\ - build-essential make wget vim dpdk libnuma-dev \\ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV DPDK_VERSION=17.11.3 \\ - RTE_SDK=/usr/src/dpdk \\ - RTE_TARGET=x86_64-native-linuxapp-gcc - -RUN wget http://fast.dpdk.org/rel/dpdk-\${DPDK_VERSION}.tar.xz && tar xf dpdk-\${DPDK_VERSION}.tar.xz && \\ - mv dpdk-stable-\${DPDK_VERSION} \${RTE_SDK} - -RUN sed -i s/CONFIG_RTE_EAL_IGB_UIO=y/CONFIG_RTE_EAL_IGB_UIO=n/ \${RTE_SDK}/config/common_linuxapp \\ - && sed -i s/CONFIG_RTE_LIBRTE_KNI=y/CONFIG_RTE_LIBRTE_KNI=n/ \${RTE_SDK}/config/common_linuxapp \\ - && sed -i s/CONFIG_RTE_KNI_KMOD=y/CONFIG_RTE_KNI_KMOD=n/ \${RTE_SDK}/config/common_linuxapp - -RUN cd \${RTE_SDK} && make install T=\${RTE_TARGET} && make -C examples - -WORKDIR \${RTE_SDK}/examples/l3fwd/\${RTE_TARGET}/app/ -EOF +kubectl create ns ovs +kubectl apply -f ./examples/ovs-vhost/userspace-ovs-netAttach-1.yaml +kubectl apply -f ./examples/ovs-vhost/testpmd-pod-1.yaml +kubectl apply -f ./examples/ovs-vhost/testpmd-pod-2.yaml ``` +At this point pod 1 should be sending traffic to pod 2. +By doing a kubectl logs on pod 2 you will be able to verify that the pod is receiving traffic by looking ath the RX value. -Build the l3fwd Docker image, tag it as `dpdk-l3fwd`: -```bash -docker build -t dpdk-l3fwd -f l3fwd-dockerfile . +Sample output from a working environment ``` +root@hostname:~/userspace-cni-network-plugin# kubectl logs -n ovs pod/ovs-app2 |tail -11 +Port statistics ==================================== + ######################## NIC statistics for port 0 ######################## + RX-packets: 404024960 RX-missed: 0 RX-bytes: 25857597440 + RX-errors: 0 + RX-nombuf: 0 + TX-packets: 0 TX-errors: 0 TX-bytes: 0 -Create the pktgen Dockerfile: -```bash -$ cat < pktgen-dockerfile -FROM debian:jessie - -RUN apt-get update && apt-get install -y --no-install-recommends \\ - gcc build-essential make wget curl git liblua5.2-dev libedit-dev libpcap-dev libncurses5-dev libncursesw5-dev pkg-config vim libnuma-dev ca-certificates \\ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV DPDK_VERSION=17.11.3 \\ - RTE_SDK=/usr/src/dpdk \\ - RTE_TARGET=x86_64-native-linuxapp-gcc \\ - PKTGEN_COMMIT=pktgen-3.4.8 \\ - PKTGEN_DIR=/usr/src/pktgen - -RUN wget http://fast.dpdk.org/rel/dpdk-\${DPDK_VERSION}.tar.xz && tar xf dpdk-\${DPDK_VERSION}.tar.xz && \\ - mv dpdk-stable-\${DPDK_VERSION} \${RTE_SDK} - -RUN sed -i s/CONFIG_RTE_EAL_IGB_UIO=y/CONFIG_RTE_EAL_IGB_UIO=n/ \${RTE_SDK}/config/common_linuxapp \\ - && sed -i s/CONFIG_RTE_LIBRTE_KNI=y/CONFIG_RTE_LIBRTE_KNI=n/ \${RTE_SDK}/config/common_linuxapp \\ - && sed -i s/CONFIG_RTE_KNI_KMOD=y/CONFIG_RTE_KNI_KMOD=n/ \${RTE_SDK}/config/common_linuxapp - -RUN sed -i s/CONFIG_RTE_APP_TEST=y/CONFIG_RTE_APP_TEST=n/ \${RTE_SDK}/config/common_linuxapp \\ - && sed -i s/CONFIG_RTE_TEST_PMD=y/CONFIG_RTE_TEST_PMD=n/ \${RTE_SDK}/config/common_linuxapp - -RUN cd \${RTE_SDK} \\ - && make install T=\${RTE_TARGET} DESTDIR=install -j + Throughput (since last show) + Rx-pps: 1044345 Rx-bps: 534705080 + Tx-pps: 0 Tx-bps: 0 + ############################################################################ -RUN git config --global user.email "root@container" \\ - && git config --global user.name "root" - -RUN git clone http://dpdk.org/git/apps/pktgen-dpdk \\ - && cd pktgen-dpdk \\ - && git checkout \${PKTGEN_COMMIT} \\ - && cd .. \\ - && mv pktgen-dpdk \${PKTGEN_DIR} - -RUN cd \${PKTGEN_DIR} \\ - && make -j - -WORKDIR \${PKTGEN_DIR}/ -EOF -``` - -Build the pktgen Docker image, tag it as`dpdk-pktgen`: -```bash -docker build -t dpdk-pktgen -f pktgen-dockerfile . -``` - -### 2. Create the pods to be used - -Create the l3fwd Pod Spec: -```bash -$ cat < l3fwd-pod.yaml -apiVersion: v1 -kind: Pod -metadata: - generateName: dpdk-l3fwd- - annotations: - k8s.v1.cni.cncf.io/networks: userspace-networkobj -spec: - containers: - - name: dpdk-l3fwd - image: dpdk-l3fwd - imagePullPolicy: IfNotPresent - securityContext: - privileged: true - runAsUser: 0 - volumeMounts: - - mountPath: /vhu/ - name: socket - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 2Gi - limits: - hugepages-1Gi: 2Gi - command: ["sleep", "infinity"] - volumes: - - name: socket - hostPath: - path: /var/lib/cni/vhostuser/ - - name: hugepage - emptyDir: - medium: HugePages - securityContext: - runAsUser: 0 - restartPolicy: Never -EOF -``` - -Create the pktgen Pod Spec: -```bash -$ cat < pktgen-pod.yaml -apiVersion: v1 -kind: Pod -metadata: - generateName: dpdk-pktgen- - annotations: - k8s.v1.cni.cncf.io/networks: userspace-networkobj -spec: - containers: - - name: dpdk-pktgen - image: dpdk-pktgen - imagePullPolicy: IfNotPresent - securityContext: - privileged: true - runAsUser: 0 - volumeMounts: - - mountPath: /vhu/ - name: socket - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 2Gi - limits: - hugepages-1Gi: 2Gi - command: ["sleep", "infinity"] - volumes: - - name: socket - hostPath: - path: /var/lib/cni/vhostuser/ - - name: hugepage - emptyDir: - medium: HugePages - securityContext: - runAsUser: 0 - restartPolicy: Never -EOF -``` - -Deploy both pods: -```bash -$ kubectl create -f l3fwd-pod.yaml -$ kubectl create -f pktgen-pod.yaml -``` - -Verify your pods are running and note their unique name IDs: -```bash -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -dpdk-l3fwd-vj4hj 1/1 Running 0 51s -dpdk-pktgen-xrsz9 1/1 Running 0 2s -``` - -### 3. Run L3FWD and Pktgen - -Using your pod ID, open a bash shell in the pktgen pod: -```bash -$ kubectl exec -it dpdk-pktgen- bash -``` - -Export the port ID prefex and start the pktgen application (adjust cores, memory, etc. to suit your system): -```bash -$ export ID=$(/vhu/get-prefix.sh) -$ ./app/x86_64-native-linuxapp-gcc/pktgen -l 10,11,12 --vdev=virtio_user0,path=/vhu/${ID}/${ID:0:12}-net1 --no-pci --socket-mem=1024,1024 --master-lcore 10 -- -m 11:12.0 -P -``` - -The pktgen application should launch. Among the statistics, make note of the pktgen source MAC address listed as ```Src MAC Address```. For example: -```bash -Src MAC Address : f2:89:22:4e:28:3b -``` - -In another terminal, open a bash shell in the l3fwd pod: -```bash -kubectl exec -it dpdk-l3fwd- bash -``` - -Export the port ID prefex and start the l3fwd application. Set the destination MAC address using the ```--eth-dest``` argument. This should be the ```Src MAC Address``` previously noted from the pktgen pod (adjust cores, memory, etc. to suit your system): -```bash -$ export ID=$(/vhu/get-prefix.sh) -$ ./l3fwd -c 0x10 --vdev=virtio_user0,path=/vhu/${ID}/${ID:0:12}-net1 --no-pci --socket-mem=1024,1024 -- -p 0x1 -P --config "(0,0,4)" --eth-dest=0, --parse-ptype -``` - -The l3fwd app should start up. Among the information printed to the screen will be the ```Address```. This is the MAC address of the l3fwd port, make note of it. - -Back on the pktgen pod, set the destination MAC address to that of the l3fwd port: -```bash -Pktgen:/> set 0 dst mac -``` - -Start traffic generation: -```bash -Pktgen:/> start 0 -``` - -You should see the packet counts for Tx and Rx increase, verifying that packets are being transmitted by pktgen and are being sent back via l3fwd running in the other pod. - -To exit: -```bash -Pktgen:/> stop 0 -Pktgen:/> quit ``` # Unit Testing @@ -1159,9 +738,3 @@ Make Targets for unit testing inside containers: make coverage-all - Calculate code coverage inside container for all supported OS distributions. e.g. make -j 5 coverage-all ``` - -Following Linux OS distributions are supported for unit testing inside containers: - -* CentOS 7, 8 -* Fedora 31, 32 -* Ubuntu 16.04, 18.04, 20.04 diff --git a/docker/dpdk-app-centos/Dockerfile b/docker/dpdk-app-centos/Dockerfile deleted file mode 100644 index 41634dcf..00000000 --- a/docker/dpdk-app-centos/Dockerfile +++ /dev/null @@ -1,94 +0,0 @@ -# To build: -# docker build --rm -t dpdk-app-centos . -# - - -# -------- Builder stage. -FROM centos:8@sha256:a27fd8080b517143cbbbab9dfb7c8571c40d67d534bbdee55bd6c473f432b177 - -# -# Install required packages -# -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - - -RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo -RUN yum groupinstall -y "Development Tools" -RUN yum install -y wget numactl-devel git golang make; yum clean all -# Debug Tools (if needed): -#RUN yum install -y pciutils iproute; yum clean all - -# -# Download and Build APP-NetUtil -# -WORKDIR /root/go/src/ -RUN go get github.com/openshift/app-netutil > /tmp/UserspaceDockerBuild.log 2>&1 || echo "Can ignore no GO files." -WORKDIR /root/go/src/github.com/openshift/app-netutil -RUN make c_sample -RUN cp bin/libnetutil_api.so /lib64/libnetutil_api.so; cp bin/libnetutil_api.h /usr/include/libnetutil_api.h - -# -# Download and Build DPDK -# -ENV DPDK_VER 19.08 -ENV DPDK_DIR /usr/src/dpdk-${DPDK_VER} -WORKDIR /usr/src/ -RUN curl --output dpdk-${DPDK_VER}.tar.xz http://fast.dpdk.org/rel/dpdk-${DPDK_VER}.tar.xz -RUN tar -xpvf dpdk-${DPDK_VER}.tar.xz - -ENV RTE_TARGET=x86_64-native-linuxapp-gcc -ENV RTE_SDK=${DPDK_DIR} -WORKDIR ${DPDK_DIR} -# DPDK_VER 19.08 -RUN sed -i -e 's/EAL_IGB_UIO=y/EAL_IGB_UIO=n/' config/common_linux -RUN sed -i -e 's/KNI_KMOD=y/KNI_KMOD=n/' config/common_linux -RUN sed -i -e 's/LIBRTE_KNI=y/LIBRTE_KNI=n/' config/common_linux -RUN sed -i -e 's/LIBRTE_PMD_KNI=y/LIBRTE_PMD_KNI=n/' config/common_linux -# Additional Debug if Needed -#RUN sed -i -e 's/CONFIG_RTE_LIBRTE_ETHDEV_DEBUG=n/CONFIG_RTE_LIBRTE_ETHDEV_DEBUG=y/' config/common_base - -# DPDK_VER 19.02 -#RUN sed -i -e 's/EAL_IGB_UIO=y/EAL_IGB_UIO=n/' config/common_linuxapp -#RUN sed -i -e 's/KNI_KMOD=y/KNI_KMOD=n/' config/common_linuxapp -#RUN sed -i -e 's/LIBRTE_KNI=y/LIBRTE_KNI=n/' config/common_linuxapp -#RUN sed -i -e 's/LIBRTE_PMD_KNI=y/LIBRTE_PMD_KNI=n/' config/common_linuxapp -RUN make install T=${RTE_TARGET} DESTDIR=${RTE_SDK} - -# -# Build TestPmd -# -WORKDIR ${DPDK_DIR}/app/test-pmd -COPY ./dpdk-args.c ./dpdk-args.c -COPY ./dpdk-args.h ./dpdk-args.h -COPY ./testpmd_eal_init.txt ./testpmd_eal_init.txt -COPY ./testpmd_launch_args_parse.txt ./testpmd_launch_args_parse.txt -COPY ./testpmd_substitute.sh ./testpmd_substitute.sh -RUN ./testpmd_substitute.sh -RUN make -RUN cp testpmd /usr/bin/testpmd -RUN cp testpmd /usr/bin/dpdk-app - -# -# Build l3fwd -# -WORKDIR ${DPDK_DIR}/examples/l3fwd -COPY ./dpdk-args.c ./dpdk-args.c -COPY ./dpdk-args.h ./dpdk-args.h -COPY ./l3fwd_eal_init.txt ./l3fwd_eal_init.txt -COPY ./l3fwd_parse_args.txt ./l3fwd_parse_args.txt -COPY ./l3fwd_substitute.sh ./l3fwd_substitute.sh -RUN ./l3fwd_substitute.sh -RUN make -RUN cp build/l3fwd /usr/bin/l3fwd -#RUN cp build/l3fwd /usr/bin/dpdk-app - -# -------- Import stage. -# Docker 17.05 or higher -##FROM centos - -# Install UserSpace CNI -##COPY --from=0 /usr/bin/dpdk-app /usr/bin/dpdk-app - -COPY ./docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["dpdk-app"] diff --git a/docker/dpdk-app-centos/README.md b/docker/dpdk-app-centos/README.md deleted file mode 100644 index a449c1a9..00000000 --- a/docker/dpdk-app-centos/README.md +++ /dev/null @@ -1,269 +0,0 @@ -# Docker Image: dpdk-app-centos -This directory contains the files needed to build a DPDK based test image. -This image is based on CentOS (latest) base image built with DPDK 19.08. - -The User Space CNI inconjunction with the OVS CNI Library (cniovs) or VPP -CNI Library (cnivpp) creates interfaces on the host, like a vhost-user or -a memif interface, adds the host side of the interface to a local network, -like a L2 bridge, then copies information needed in the container into -annotations. The container, like this one, boots up and reads the -annotatations and runs a DPDK application. - - -# Build Instructions for dpdk-app-centos Docker Image -Get the **user-space-net-plugin** repo: -``` - cd $GOPATH/src/ - go get github.com/intel/userspace-cni-network-plugin -``` - -Build the docker image: -``` - cd $GOPATH/src/github.com/intel/userspace-cni-network-plugin/docker/dpdk-app-centos/ - docker build --rm -t dpdk-app-centos . -``` - -## Reduce Image Size -Multi-stage builds are a new feature requiring **Docker 17.05** or higher on -the daemon and client. If multi-stage builds are supported on your system, -then uncomment the following lines (those with **##**) in the Dockerfile: -``` -: - -# -------- Import stage. -# Docker 17.05 or higher -##FROM centos - -# Install UserSpace CNI -##COPY --from=0 /usr/bin/dpdk-app /usr/bin/dpdk-app - -: -``` - -**NOTE:** Need to verify this works as designed. System currently has older -version of Docker and haven't been able to verify. May need to add an -additional package or two to the image after the import stage. - - -# Docker Image Details -This Docker Image is downloading DPDK (version 19.08 to get memif PMD) -and building it. Once built, changing into the DPDK `testpmd` -directory (${DPDK_DIR}/app/test-pmd) and building it. - -`testpmd` is a sample DPDK application that comes with DPDK. Typically, -`testpmd` is started with a set of input parameters that initializes DPDK. -For example: -``` -$ testpmd \ --m 1024 \ --l 2-3 \ --n 4 \ ---vdev=virtio_user0,path=/var/lib/cni/usrspcni/558cbb292167-net1 \ ---vdev=virtio_user1,path=/var/lib/cni/usrspcni/558cbb292167-net2 \ ---no-pci \ --- \ ---auto-start \ ---tx-first \ ---no-lsc-interrupt \ ---stats-period 10 -``` - -This Docker image is tweaking this a little. Before `testpmd` is built, the -testpmd.c file (contains main()) is updated using 'sed'. See -'testpmd_substitute.sh'. - -**NOTE:** If a different version of DPDK is needed or used, this local file -may need to be synchronized with the updated version. - -An additional file, dpdk-args.c, is also added to the directory and Makefile. -The changes to testpmd.c are simply to call a function in dpdk-args.c which -will generate this list of input parameters, and then pass this private set -of parameters to DPDK functions instead of the inpupt `argc` and `argv`. When -the generated binary is copied to `/usr/bin/`, it is renamed to `dpdk-app`. - -The code is leveraging another project, app-netutil -(https://github.com/openshift/app-netutil), which is a library design to be -called within a container to collect all the configuration data, like that -stored in annotations by Userspace CNI, and expose it to a DPDK application -in a clean API. - -**NOTE:** For debugging, if `dpdk-app` is called with a set of input parameters, -it will skip the dpdk-args.c code and behave exactly as `testpmd`. Just add -the `sleep` to the pod spec: -``` -: - resources: - requests: - memory: 2Mi - limits: - hugepages-2Mi: 1024Mi - command: ["sleep", "infinity"] <-- ADD - nodeSelector: - vswitch: ovs -: -``` - -Then get a pod shell: -``` - kubectl exec -it userspace-ovs-pod-1 -- sh -``` - -Run `dpdk-app` with no parameters, and it will be as if it is called -as the container is started. It also prints out the generated parameter -list, which include the dynamic socketfile path: -``` -sh-4.2# dpdk-app -COLLECT Data: - cpuRsp.CPUSet = 0-63 - Interface[0]: - IfName="eth0" Name="cbr0" Type=unknown - MAC="5e:6b:7e:19:5b:94" IP="10.244.0.197" - Interface[1]: - IfName="net1" Name="sriov-network-a" Type=SR-IOV - MAC="" - PCIAddress=0000:01:0a.4 - Interface[2]: - IfName="net2" Name="sriov-network-b" Type=SR-IOV - MAC="" - PCIAddress=0000:01:02.4 -ENTER dpdk-app (testpmd): - myArgc=15 - dpdk-app -n 4 -l 1-3 --master-lcore 1 --vdev=virtio_user0,path=/var/lib/cni/usrspcni/34c8ba49b767-net1 --vdev=virtio_user1,path=/var/lib/cni/usrspcni/34c8ba49b767-net2 --no-pci -- --auto-start --tx-first --no-lsc-interrupt --stats-period 60 -EAL: Detected 64 lcore(s) -EAL: Detected 2 NUMA nodes -EAL: Multi-process socket /var/run/dpdk/rte/mp_socket -: -``` - -Then 'CTRL-C' to exit and re-run `dpdk-app` with input parameters -modified as needed: -``` -dpdk-app \ --l 1-3 \ --master-lcore 1 \ --n 4 \ ---vdev=virtio_user0,path=/var/lib/cni/usrspcni/34c8ba49b767-net1 \ ---vdev=virtio_user1,path=/var/lib/cni/usrspcni/34c8ba49b767-net2 \ ---no-pci \ --- \ ---auto-start \ ---tx-first \ ---no-lsc-interrupt -``` - - -# Deploy Image -An example of using this Docker image can be found in this same repo under: -``` - $GOPATH/src/github.com/intel/userspace-cni-network-plugin/examples/ovs-host/ -``` - -This example assumes that kubernetes has already been deployed with `multus` -and some CNI for the default network (like flannel). It also assumes that OvS -is running on one or more of the worker nodes and the worker nodes running -OvS have the label `vswitch=ovs` applied. For example, for the node -`k8s-work-1-f29-ovs1`, run: -``` - kubectl label nodes k8s-work-1-f29-ovs1 vswitch=ovs -``` - -The files in this subdirectory are used to create two networks: -* userspace-ovs-net-1 - * Creates bridge `br-4` on the local OvS instance - * Adds vhost interface [ContainerId:12]-net[instance] (i.e. 8a0dd2a77c59-net1) - to the bridge on the local OvS instance. - * Adds IP in the 10.56.217.0/24 subnet to the vhost interface in the container. -* userspace-ovs-net-2 - * Creates bridge `br-5` on the local OvS instance - * Adds vhost interface [ContainerId:12]-net[instance] (i.e. 8a0dd2a77c59-net2) - to the bridge on the local OvS instance. - * Adds IP in the 10.77.217.0/24 subnet to the vhost interface in container. - -To apply the CRD for these networks: -``` - kubectl create -f examples/ovs-vhost/userspace-ovs-netAttach-1.yaml - kubectl create -f examples/ovs-vhost/userspace-ovs-netAttach-2.yaml -``` - -Create two pods, each with 3 interfaces: -* 1 on the default network (i.e. kernel interface using flannel) -* 1 on userspace-ovs-net-1 -* 1 on userspace-ovs-net-2 - -**NOTE:** Userspace CNI does not manage node to node networking. So -in this example, the two pods must be created on the same node (the -reason for the `vswitch=ovs` label), or OvS on the different nodes -must have connectivity setup between them. - -To create the two pods: -``` - kubectl create -f examples/ovs-vhost/userspace-ovs-pod-1.yaml - kubectl create -f examples/ovs-vhost/userspace-ovs-pod-2.yaml -``` - -To verify setup, on host, inspect OvS configuration: -``` -sudo ovs-vsctl show -26373aec-1484-4980-8853-9db2183cfafc - Bridge "br-5" - Port "8a0dd2a77c59-net2" - Interface "8a0dd2a77c59-net2" - type: dpdkvhostuser - Port "e5b2972a8210-net2" - Interface "e5b2972a8210-net2" - type: dpdkvhostuser - Port "br-5" - Interface "br-5" - type: internal - Bridge "br-4" - Port "br-4" - Interface "br-4" - type: internal - Port "8a0dd2a77c59-net1" - Interface "8a0dd2a77c59-net1" - type: dpdkvhostuser - Port "e5b2972a8210-net1" - Interface "e5b2972a8210-net1" - type: dpdkvhostuser - ovs_version: "2.9.2" -``` - -To show statistics (choose an interface form above output): -``` -sudo ovs-vsctl list interface 8a0dd2a77c59-net2 -[sudo] password for bmcfall: -_uuid : dde9399d-88fc-43c2-b3a1-8073fa3f9cef -admin_state : up -bfd : {} -bfd_status : {} -cfm_fault : [] -cfm_fault_status : [] -cfm_flap_count : [] -cfm_health : [] -cfm_mpid : [] -cfm_remote_mpids : [] -cfm_remote_opstate : [] -duplex : [] -error : [] -external_ids : {} -ifindex : 15730733 -ingress_policing_burst: 0 -ingress_policing_rate: 0 -lacp_current : [] -link_resets : 0 -link_speed : [] -link_state : up -lldp : {} -mac : [] -mac_in_use : "00:00:00:00:00:00" -mtu : 1500 -mtu_request : [] -name : "8a0dd2a77c59-net2" -ofport : 2 -ofport_request : [] -options : {} -other_config : {} -statistics : {"rx_1024_to_1522_packets"=0, "rx_128_to_255_packets"=0, "rx_1523_to_max_packets"=0, "rx_1_to_64_packets"=4226816, "rx_256_to_511_packets"=0, "rx_512_to_1023_packets"=0, "rx_65_to_127_packets"=0, rx_bytes=270516224, rx_dropped=0, rx_errors=0, rx_packets=4226816, tx_bytes=270514176, tx_dropped=0, tx_packets=4226784} -status : {features="0x0000000110008000", mode=server, num_of_vrings="2", numa="0", socket="/usr/local/var/run/openvswitch/8a0dd2a77c59-net2", status=connected, "vring_0_size"="256", "vring_1_size"="256"} -type : dpdkvhostuser -``` \ No newline at end of file diff --git a/docker/dpdk-app-centos/docker-entrypoint.sh b/docker/dpdk-app-centos/docker-entrypoint.sh deleted file mode 100755 index a6453106..00000000 --- a/docker/dpdk-app-centos/docker-entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -exec "$@" diff --git a/docker/dpdk-app-centos/dpdk-args.c b/docker/dpdk-app-centos/dpdk-args.c deleted file mode 100644 index 46cdac85..00000000 --- a/docker/dpdk-app-centos/dpdk-args.c +++ /dev/null @@ -1,437 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2019 Red Hat - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "libnetutil_api.h" -#include "dpdk-args.h" - -bool debugArgs = true; - -#define DPDK_ARGS_MAX_ARGS (30) -#define DPDK_ARGS_MAX_ARG_STRLEN (100) -char myArgsArray[DPDK_ARGS_MAX_ARGS][DPDK_ARGS_MAX_ARG_STRLEN]; -char* myArgv[DPDK_ARGS_MAX_ARGS]; - -//#define DPDK_ARGS_MAX_NUM_DIR (30) -//static const char DEFAULT_DIR[] = "/var/lib/cni/"; - -static char STR_MASTER[] = "master"; -static char STR_SLAVE[] = "slave"; -static char STR_ETHERNET[] = "ethernet"; - -/* Large enough to hold: ",mac=aa:bb:cc:dd:ee:ff" */ -#define DPDK_ARGS_MAX_MAC_STRLEN (25) - - -static int getInterfaces(int argc, int *pPortCnt, int *pPortMask) { - int i = 0; - int j; - int vhostCnt = 0; - int memifCnt = 1; - int sriovCnt = 0; - int err; - struct InterfaceResponse ifaceRsp; - char macStr[DPDK_ARGS_MAX_MAC_STRLEN]; -#if 0 - // Refactor code later to find JSON file. Needs work so commented out for now. - DIR *d; - struct dirent *dir; - int currIndex = 0; - int freeIndex = 0; - char* dirList[DPDK_ARGS_MAX_NUM_DIR]; - char* fileExt; - - memset(dirList, 0, sizeof(char*)*DPDK_ARGS_MAX_NUM_DIR); - - dirList[freeIndex] = malloc(sizeof(char) * (strlen(DEFAULT_DIR)+1)); - strcpy(dirList[freeIndex++], DEFAULT_DIR); - - while (dirList[currIndex] != NULL) { - printf(" Directory:%s\n", dirList[currIndex]); - d = opendir(dirList[currIndex]); - if (d) - { - while ((dir = readdir(d)) != NULL) - { - if ((dir->d_name) && - (strcmp(dir->d_name, ".") != 0) && - (strcmp(dir->d_name, "..") != 0)) - { - printf(" Name:%s %d\n", dir->d_name, dir->d_type); - if (dir->d_type == DT_DIR) { - printf(" Add to Dir List:%s\n", dir->d_name); - dirList[freeIndex] = malloc(sizeof(char) * (strlen(DEFAULT_DIR)+strlen(dir->d_name)+1)); - sprintf(dirList[freeIndex++], "%s%s/", DEFAULT_DIR, dir->d_name); - } - else - { - if (strstr(dir->d_name, "net") != NULL) - { - fileExt = strrchr(dir->d_name, '.'); - if ((fileExt == NULL) || (strcmp(fileExt, ".json") != 0)) { - printf(" Adding to vdev list:%s\n", dir->d_name); - snprintf(&myArgsArray[argc++][0], DPDK_ARGS_MAX_ARG_STRLEN-1, - "--vdev=virtio_user%d,path=%s%s", i, dirList[currIndex], dir->d_name); - i++; - } - else { - printf(" Invalid FileExt\n"); - } - } - } - } - } - closedir(d); - } - free(dirList[currIndex]); - currIndex++; - } -#endif - - ifaceRsp.numIfaceAllocated = NETUTIL_NUM_NETWORKINTERFACE; - ifaceRsp.numIfacePopulated = 0; - ifaceRsp.pIface = malloc(ifaceRsp.numIfaceAllocated * sizeof(struct InterfaceData)); - if (ifaceRsp.pIface) { - memset(ifaceRsp.pIface, 0, (ifaceRsp.numIfaceAllocated * sizeof(struct InterfaceData))); - err = GetInterfaces(&ifaceRsp); - if ((err == NETUTIL_ERRNO_SUCCESS) || (err == NETUTIL_ERRNO_SIZE_ERROR)) { - for (i = 0; i < ifaceRsp.numIfacePopulated; i++) { - if (debugArgs) { - printf(" Interface[%d]:\n", i); - - printf(" "); - if (ifaceRsp.pIface[i].IfName) { - printf(" IfName=\"%s\"", ifaceRsp.pIface[i].IfName); - } - if (ifaceRsp.pIface[i].Name) { - printf(" Name=\"%s\"", ifaceRsp.pIface[i].Name); - } - printf(" Type=%s", - (ifaceRsp.pIface[i].Type == NETUTIL_TYPE_KERNEL) ? "kernel" : - (ifaceRsp.pIface[i].Type == NETUTIL_TYPE_SRIOV) ? "SR-IOV" : - (ifaceRsp.pIface[i].Type == NETUTIL_TYPE_VHOST) ? "vHost" : - (ifaceRsp.pIface[i].Type == NETUTIL_TYPE_MEMIF) ? "memif" : - (ifaceRsp.pIface[i].Type == NETUTIL_TYPE_VDPA) ? "vDPA" : - (ifaceRsp.pIface[i].Type == NETUTIL_TYPE_UNKNOWN) ? "unknown" : "error"); - printf("\n"); - - printf(" "); - if (ifaceRsp.pIface[i].Network.Mac) { - printf(" MAC=\"%s\"", ifaceRsp.pIface[i].Network.Mac); - } - for (j = 0; j < NETUTIL_NUM_IPS; j++) { - if (ifaceRsp.pIface[i].Network.IPs[j]) { - printf(" IP=\"%s\"", ifaceRsp.pIface[i].Network.IPs[j]); - } - } - printf("\n"); - } - - switch (ifaceRsp.pIface[i].Type) { - case NETUTIL_TYPE_SRIOV: - if (debugArgs) { - printf(" "); - if (ifaceRsp.pIface[i].Sriov.PCIAddress) { - printf(" PCIAddress=%s", ifaceRsp.pIface[i].Sriov.PCIAddress); - } - printf("\n"); - } - - if (ifaceRsp.pIface[i].Sriov.PCIAddress) { - snprintf(&myArgsArray[argc++][0], DPDK_ARGS_MAX_ARG_STRLEN-1, - "-w %s", ifaceRsp.pIface[i].Sriov.PCIAddress); - sriovCnt++; - - free(ifaceRsp.pIface[i].Sriov.PCIAddress); - - *pPortMask = *pPortMask | 1 << *pPortCnt; - *pPortCnt = *pPortCnt + 1; - } - break; - case NETUTIL_TYPE_VHOST: - if (debugArgs) { - printf(" "); - printf(" Mode=%s", - (ifaceRsp.pIface[i].Vhost.Mode == NETUTIL_VHOST_MODE_CLIENT) ? "client" : - (ifaceRsp.pIface[i].Vhost.Mode == NETUTIL_VHOST_MODE_SERVER) ? "server" : "error"); - if (ifaceRsp.pIface[i].Vhost.Socketpath) { - printf(" Socketpath=\"%s\"", ifaceRsp.pIface[i].Vhost.Socketpath); - } - printf("\n"); - } - - if (ifaceRsp.pIface[i].Vhost.Socketpath) { - if (ifaceRsp.pIface[i].Vhost.Mode == NETUTIL_VHOST_MODE_SERVER) { - snprintf(&myArgsArray[argc++][0], DPDK_ARGS_MAX_ARG_STRLEN-1, - "--vdev=virtio_user%d,path=%s", vhostCnt, ifaceRsp.pIface[i].Vhost.Socketpath); - - vhostCnt++; - *pPortMask = *pPortMask | 1 << *pPortCnt; - *pPortCnt = *pPortCnt + 1; - } - else if (ifaceRsp.pIface[i].Vhost.Mode == NETUTIL_VHOST_MODE_CLIENT) { - snprintf(&myArgsArray[argc++][0], DPDK_ARGS_MAX_ARG_STRLEN-1, - "--vdev=virtio_user%d,path=%s,queues=1", vhostCnt, ifaceRsp.pIface[i].Vhost.Socketpath); - - vhostCnt++; - *pPortMask = *pPortMask | 1 << *pPortCnt; - *pPortCnt = *pPortCnt + 1; - } else { - printf("ERROR: Unknown vHost Mode=%d\n", ifaceRsp.pIface[i].Vhost.Mode); - } - free(ifaceRsp.pIface[i].Vhost.Socketpath); - } - break; - case NETUTIL_TYPE_MEMIF: - if (debugArgs) { - printf(" "); - printf(" Role=%s", - (ifaceRsp.pIface[i].Memif.Role == NETUTIL_MEMIF_ROLE_MASTER) ? "master" : - (ifaceRsp.pIface[i].Memif.Role == NETUTIL_MEMIF_ROLE_SLAVE) ? "slave" : "error"); - printf(" Mode=%s", - (ifaceRsp.pIface[i].Memif.Mode == NETUTIL_MEMIF_MODE_ETHERNET) ? "ethernet" : - (ifaceRsp.pIface[i].Memif.Mode == NETUTIL_MEMIF_MODE_IP) ? "ip" : - (ifaceRsp.pIface[i].Memif.Mode == NETUTIL_MEMIF_MODE_INJECT_PUNT) ? "inject-punt" : "error"); - if (ifaceRsp.pIface[i].Memif.Socketpath) { - printf(" Socketpath=\"%s\"", ifaceRsp.pIface[i].Memif.Socketpath); - } - printf("\n"); - } - - if (ifaceRsp.pIface[i].Memif.Socketpath) { - char *pRole = NULL; - char *pMode = NULL; - - if (ifaceRsp.pIface[i].Memif.Role == NETUTIL_MEMIF_ROLE_MASTER) { - pRole = STR_MASTER; - } - else if (ifaceRsp.pIface[i].Memif.Role == NETUTIL_MEMIF_ROLE_SLAVE) { - pRole = STR_SLAVE; - } - else { - printf("ERROR: Unknown memif Role=%d\n", ifaceRsp.pIface[i].Memif.Role); - } - - if (ifaceRsp.pIface[i].Memif.Mode == NETUTIL_MEMIF_MODE_ETHERNET) { - pMode = STR_ETHERNET; - } - else if (ifaceRsp.pIface[i].Memif.Mode == NETUTIL_MEMIF_MODE_IP) { - //pMode = "ip"; - printf("ERROR: memif Mode=%d - Not Supported in DPDK!\n", ifaceRsp.pIface[i].Memif.Mode); - } - else if (ifaceRsp.pIface[i].Memif.Mode == NETUTIL_MEMIF_MODE_INJECT_PUNT) { - //pMode = "inject-punt""; - printf("ERROR: memif Mode=%d - Not Supported in DPDK!\n", ifaceRsp.pIface[i].Memif.Mode); - } - else { - printf("ERROR: Unknown memif Mode=%d\n", ifaceRsp.pIface[i].Memif.Mode); - } - - if ((ifaceRsp.pIface[i].Network.Mac) && - (strcmp(ifaceRsp.pIface[i].Network.Mac,"") != 0)) { - snprintf(&macStr[0], DPDK_ARGS_MAX_MAC_STRLEN-1, - ",mac=%s", ifaceRsp.pIface[i].Network.Mac); - } - else { - macStr[0] = '\0'; - } - - if ((pRole) && (pMode)) { - snprintf(&myArgsArray[argc++][0], DPDK_ARGS_MAX_ARG_STRLEN-1, - "--vdev=net_memif%d,socket=%s,role=%s%s", memifCnt, ifaceRsp.pIface[i].Memif.Socketpath, pRole, &macStr[0]); - - memifCnt++; - *pPortMask = *pPortMask | 1 << *pPortCnt; - *pPortCnt = *pPortCnt + 1; - } - - free(ifaceRsp.pIface[i].Memif.Socketpath); - } - break; - } - - if (ifaceRsp.pIface[i].Network.Mac) { - free(ifaceRsp.pIface[i].Network.Mac); - } - for (j = 0; j < NETUTIL_NUM_IPS; j++) { - if (ifaceRsp.pIface[i].Network.IPs[j]) { - free(ifaceRsp.pIface[i].Network.IPs[j]); - } - } - - if (ifaceRsp.pIface[i].IfName) { - free(ifaceRsp.pIface[i].IfName); - } - if (ifaceRsp.pIface[i].Name) { - free(ifaceRsp.pIface[i].Name); - } - } /* END of FOR EACH Interface */ - - - if (sriovCnt == 0) { - strncpy(&myArgsArray[argc++][0], "--no-pci", DPDK_ARGS_MAX_ARG_STRLEN-1); - } - } - else { - printf("Couldn't get network interface, err code: %d\n", err); - } - } - - return(argc); -} - -char** GetArgs(int *pArgc, eDpdkAppType appType) -{ - int argc = 0; - int i; - struct CPUResponse cpuRsp; - int err; - int portMask = 0; - int portCnt = 0; - int lcoreBase = 0; - int port; - int length = 0; - - sleep(2); - - memset(&cpuRsp, 0, sizeof(cpuRsp)); - err = GetCPUInfo(&cpuRsp); - if (err) { - printf("Couldn't get CPU info, err code: %d\n", err); - } - if (cpuRsp.CPUSet) { - printf(" cpuRsp.CPUSet = %s\n", cpuRsp.CPUSet); - - // Free the string - free(cpuRsp.CPUSet); - } - - - memset(&myArgsArray[0][0], 0, sizeof(char)*DPDK_ARGS_MAX_ARG_STRLEN*DPDK_ARGS_MAX_ARGS); - memset(&myArgv[0], 0, sizeof(char)*DPDK_ARGS_MAX_ARGS); - - if (pArgc) { - /* - * Initialize EAL Options - */ - strncpy(&myArgsArray[argc++][0], "dpdk-app", DPDK_ARGS_MAX_ARG_STRLEN-1); - - //strncpy(&myArgsArray[argc++][0], "-m", DPDK_ARGS_MAX_ARG_STRLEN-1); - //strncpy(&myArgsArray[argc++][0], "1024", DPDK_ARGS_MAX_ARG_STRLEN-1); - - strncpy(&myArgsArray[argc++][0], "-n", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "4", DPDK_ARGS_MAX_ARG_STRLEN-1); - - //strncpy(&myArgsArray[argc++][0], "--file-prefix=dpdk-app_", DPDK_ARGS_MAX_ARG_STRLEN-1); - - if (appType == DPDK_APP_TESTPMD) { - strncpy(&myArgsArray[argc++][0], "-l", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "1-3", DPDK_ARGS_MAX_ARG_STRLEN-1); - - strncpy(&myArgsArray[argc++][0], "--master-lcore", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "1", DPDK_ARGS_MAX_ARG_STRLEN-1); - - argc = getInterfaces(argc, &portCnt, &portMask); - - /* - * Initialize APP Specific Options - */ - strncpy(&myArgsArray[argc++][0], "--", DPDK_ARGS_MAX_ARG_STRLEN-1); - - strncpy(&myArgsArray[argc++][0], "--auto-start", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "--tx-first", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "--no-lsc-interrupt", DPDK_ARGS_MAX_ARG_STRLEN-1); - - /* testpmd exits if there is not user enteraction, so print stats */ - /* every so often to keep program running. */ - strncpy(&myArgsArray[argc++][0], "--stats-period", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "60", DPDK_ARGS_MAX_ARG_STRLEN-1); - } - else if (appType == DPDK_APP_L3FWD) { - /* NOTE: The l3fwd app requires a TX Queue per lcore. So seeting lcore to 1 */ - /* until additional queues are added to underlying interface. */ - strncpy(&myArgsArray[argc++][0], "-l", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "1", DPDK_ARGS_MAX_ARG_STRLEN-1); - - strncpy(&myArgsArray[argc++][0], "--master-lcore", DPDK_ARGS_MAX_ARG_STRLEN-1); - strncpy(&myArgsArray[argc++][0], "1", DPDK_ARGS_MAX_ARG_STRLEN-1); - lcoreBase = 1; - - argc = getInterfaces(argc, &portCnt, &portMask); - - /* - * Initialize APP Specific Options - */ - strncpy(&myArgsArray[argc++][0], "--", DPDK_ARGS_MAX_ARG_STRLEN-1); - - /* Set the PortMask, Hexadecimal bitmask of ports used by app. */ - strncpy(&myArgsArray[argc++][0], "-p", DPDK_ARGS_MAX_ARG_STRLEN-1); - snprintf(&myArgsArray[argc++][0], DPDK_ARGS_MAX_ARG_STRLEN-1, - "0x%x", portMask); - - /* Set all ports to promiscuous mode so that packets are accepted */ - /* regardless of the packet’s Ethernet MAC destination address. */ - strncpy(&myArgsArray[argc++][0], "-P", DPDK_ARGS_MAX_ARG_STRLEN-1); - -#if 1 - /* Determines which queues from which ports are mapped to which cores. */ - /* Usage: --config="(port,queue,lcore)[,(port,queue,lcore)]" */ - length = 0; - length += snprintf(&myArgsArray[argc][length], DPDK_ARGS_MAX_ARG_STRLEN-length, - "--config=\""); - for (port = 0; port < portCnt; port++) { - /* If not the first port, add a ',' to string. */ - if (port != 0) { - length += snprintf(&myArgsArray[argc][length], DPDK_ARGS_MAX_ARG_STRLEN-length, ","); - } - - /* Add each port data */ - length += snprintf(&myArgsArray[argc][length], DPDK_ARGS_MAX_ARG_STRLEN-length, - "(%d,%d,%d)", port, 0 /* queue */, lcoreBase /*+port*/); - - /* If the last port, add a trailing " to string. */ - if (port == portCnt-1) { - length += snprintf(&myArgsArray[argc][length], DPDK_ARGS_MAX_ARG_STRLEN-length, "\""); - } - } - argc++; -#else - /* Determines which queues from which ports are mapped to which cores. */ - /* Usage: --config (port,queue,lcore)[,(port,queue,lcore)] */ - strncpy(&myArgsArray[argc++][0], "--config", DPDK_ARGS_MAX_ARG_STRLEN-1); - length = 0; - for (port = 0; port < portCnt; port++) { - /* If not the first port, add a ',' to string. */ - if (port != 0) { - length += snprintf(&myArgsArray[argc][length], DPDK_ARGS_MAX_ARG_STRLEN-length, ","); - } - - /* Add each port data */ - length += snprintf(&myArgsArray[argc][length], DPDK_ARGS_MAX_ARG_STRLEN-length, - "(%d,%d,%d)", port, 0 /* queue */, lcoreBase /*+port*/); - } - argc++; -#endif - - /* Set to use software to analyze packet type. Without this option, */ - /* hardware will check the packet type. Not sure if vHost supports. */ - strncpy(&myArgsArray[argc++][0], "--parse-ptype", DPDK_ARGS_MAX_ARG_STRLEN-1); - - } - - for (i = 0; i < argc; i++) { - myArgv[i] = &myArgsArray[i][0]; - } - *pArgc = argc; - } - - return(myArgv); -} \ No newline at end of file diff --git a/docker/dpdk-app-centos/dpdk-args.h b/docker/dpdk-app-centos/dpdk-args.h deleted file mode 100644 index f1dbd91c..00000000 --- a/docker/dpdk-app-centos/dpdk-args.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2019 Red Hat - */ - -#ifndef __DPDK_ARGS_H__ -#define __DPDK_ARGS_H__ - -typedef enum { - DPDK_APP_TESTPMD = 1, - DPDK_APP_L3FWD, - DPDK_APP_OTHER -} eDpdkAppType; - -extern char** GetArgs(int *pArgc, eDpdkAppType appType); - -#endif /* __DPDK_ARGS_H__ */ diff --git a/docker/dpdk-app-centos/l3fwd_eal_init.txt b/docker/dpdk-app-centos/l3fwd_eal_init.txt deleted file mode 100644 index 1b345b38..00000000 --- a/docker/dpdk-app-centos/l3fwd_eal_init.txt +++ /dev/null @@ -1,41 +0,0 @@ -#if 0 - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); - argc -= ret; - argv += ret; -#else - int i; - int myArgc = 0; - char **myArgv = NULL; - - if (argc > 1) { - printf("ENTER dpdk-app (l3fwd):\n"); - printf(" argc=%d\n ", argc); - for (i = 0; i < argc; i++) { - printf(" %s", argv[i]); - } - printf("\n"); - - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); - argc -= ret; - argv += ret; - } else { - printf("COLLECT Data:\n"); - myArgv = GetArgs(&myArgc, DPDK_APP_L3FWD); - printf("ENTER dpdk-app (l3fwd):\n"); - printf(" myArgc=%d\n ", myArgc); - for (i = 0; i < myArgc; i++) { - printf(" %s", myArgv[i]); - } - printf("\n"); - - ret = rte_eal_init(myArgc, myArgv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); - myArgc -= ret; - myArgv += ret; - } -#endif diff --git a/docker/dpdk-app-centos/l3fwd_parse_args.txt b/docker/dpdk-app-centos/l3fwd_parse_args.txt deleted file mode 100644 index 57b4e8d3..00000000 --- a/docker/dpdk-app-centos/l3fwd_parse_args.txt +++ /dev/null @@ -1,9 +0,0 @@ -#if 0 - ret = parse_args(argc, argv); -#else - if (argc > 1) { - ret = parse_args(argc, argv); - } else { - ret = parse_args(myArgc, myArgv); - } -#endif diff --git a/docker/dpdk-app-centos/l3fwd_substitute.sh b/docker/dpdk-app-centos/l3fwd_substitute.sh deleted file mode 100755 index c03020af..00000000 --- a/docker/dpdk-app-centos/l3fwd_substitute.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -sed -i -e '/#include "l3fwd.h"/a #include "dpdk-args.h"' main.c - -sed -i -e 's!.offloads = DEV_RX_OFFLOAD_CHECKSUM,!.offloads = 0, /*DEV_RX_OFFLOAD_CHECKSUM,*/!' main.c - -sed -i '/ret = rte_eal_init(argc, argv);/{ -:a;N;/argv += ret;/!ba;N;s/.*\n//g -r l3fwd_eal_init.txt -}' main.c - -sed -i '/ret = parse_args(argc, argv);/{ -s/ret = parse_args(argc, argv);//g -r l3fwd_parse_args.txt -}' main.c - -sed -i -e '/SRCS-y :=/a SRCS-y += dpdk-args.c' Makefile -sed -i -e '/SRCS-y += dpdk-args.c/a LDLIBS += -lnetutil_api' Makefile diff --git a/docker/dpdk-app-centos/testpmd_eal_init.txt b/docker/dpdk-app-centos/testpmd_eal_init.txt deleted file mode 100644 index 1e048463..00000000 --- a/docker/dpdk-app-centos/testpmd_eal_init.txt +++ /dev/null @@ -1,30 +0,0 @@ -#if 0 - diag = rte_eal_init(argc, argv); -#else - int i; - int myArgc = 0; - char **myArgv = NULL; - - if (argc > 1) { - printf("ENTER dpdk-app (testpmd):\n"); - printf(" argc=%d\n ", argc); - for (i = 0; i < argc; i++) { - printf(" %s", argv[i]); - } - printf("\n"); - - diag = rte_eal_init(argc, argv); - } else { - printf("COLLECT Data:\n"); - myArgv = GetArgs(&myArgc, DPDK_APP_TESTPMD); - printf("ENTER dpdk-app (testpmd):\n"); - printf(" myArgc=%d\n ", myArgc); - for (i = 0; i < myArgc; i++) { - printf(" %s", myArgv[i]); - } - printf("\n"); - - diag = rte_eal_init(myArgc, myArgv); - } -#endif - diff --git a/docker/dpdk-app-centos/testpmd_launch_args_parse.txt b/docker/dpdk-app-centos/testpmd_launch_args_parse.txt deleted file mode 100644 index 7c2254bf..00000000 --- a/docker/dpdk-app-centos/testpmd_launch_args_parse.txt +++ /dev/null @@ -1,19 +0,0 @@ -#if 0 - argc -= diag; - argv += diag; - if (argc > 1) - launch_args_parse(argc, argv); -#else - if (argc > 1) { - argc -= diag; - argv += diag; - if (argc > 1) - launch_args_parse(argc, argv); - } else { - myArgc -= diag; - myArgv += diag; - if (myArgc > 1) - launch_args_parse(myArgc, myArgv); - } -#endif - diff --git a/docker/dpdk-app-centos/testpmd_substitute.sh b/docker/dpdk-app-centos/testpmd_substitute.sh deleted file mode 100755 index 8e45c55e..00000000 --- a/docker/dpdk-app-centos/testpmd_substitute.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -sed -i -e '/#include "testpmd.h"/a #include "dpdk-args.h"' testpmd.c - -sed -i '/diag = rte_eal_init(argc, argv);/{ -s/diag = rte_eal_init(argc, argv);//g -r testpmd_eal_init.txt -}' testpmd.c - -sed -i '/argc -= diag;/{ -:a;N;/launch_args_parse(argc, argv);/!ba;N;s/.*\n//g -r testpmd_launch_args_parse.txt -}' testpmd.c - -sed -i -e 's/SRCS-y += parameters.c/SRCS-y += parameters.c dpdk-args.c/' Makefile - -sed -i -e '/SRCS-y += util.c/a LDLIBS += -lnetutil_api' Makefile - diff --git a/docker/testpmd/Dockerfile b/docker/testpmd/Dockerfile new file mode 100644 index 00000000..94f9e600 --- /dev/null +++ b/docker/testpmd/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:22.04 +RUN apt-get -q update && apt-get install -y git vim meson python3-pyelftools libnuma-dev +RUN git clone https://github.com/DPDK/dpdk.git +WORKDIR /dpdk/ +RUN meson build +RUN cd build && ninja +WORKDIR /dpdk/build/app +COPY ./testpmd.sh testpmd.sh +CMD ./testpmd.sh diff --git a/docker/testpmd/testpmd.sh b/docker/testpmd/testpmd.sh new file mode 100755 index 00000000..e82e8a92 --- /dev/null +++ b/docker/testpmd/testpmd.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +cd /dpdk || exit 1 + +#set container id as env +LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/x86_64-linux-gnu/ +while ! [ -s /etc/podinfo/annotations ]; do + echo "annotations not found yet" + sleep 1 # throttle the check +done +cat /etc/podinfo/annotations +sleep 10 +containerid=$(grep containerId /etc/podinfo/annotations |cut -d '"' -f 5 |sed 's/\\//') +echo "${containerid:0:12}" + +if grep -q app1 /etc/podinfo/labels; then + fordwardmode="txonly" + cpu="2,3,4,5" +else + fordwardmode="rxonly" + cpu="6,7,8,9" +fi + +#--stats-period 1 is needed to avoid testpmd exiting +commands="./build/app/dpdk-testpmd -l $cpu --vdev net_virtio_user0,path=/var/lib/cni/usrspcni/${containerid:0:12}-net1,server=1,queue_size=2048 --in-memory --single-file-segments -- --tx-ip 192.168.1.1,192.168.1.2 --tx-udp=4000,4000 --forward-mode=$fordwardmode --stats-period 1" #-i +echo "$commands" +$commands diff --git a/docker/unit-tests/Dockerfile.base.in b/docker/unit-tests/Dockerfile.base.in deleted file mode 100644 index 741409c4..00000000 --- a/docker/unit-tests/Dockerfile.base.in +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// install GO version set by argument or fallback to default ver. -ARG UT_GO_VERSION=1.14.3 -RUN wget "https://dl.google.com/go/go${UT_GO_VERSION}.linux-amd64.tar.gz" && \\ - tar -C "/usr/local" -xzf "go${UT_GO_VERSION}.linux-amd64.tar.gz" && \\ - rm "go${UT_GO_VERSION}.linux-amd64.tar.gz" -ENV PATH=$PATH:/usr/local/go/bin -ENV GOPATH="/root/go" -ENV GOROOT="/usr/local/go" -RUN mkdir $GOPATH - -// install unit test specific go modules -RUN go get -u "github.com/msoap/go-carpet" -RUN ln -s $(go env GOPATH)/bin/go-carpet /usr/local/bin/go-carpet - -// copy userspace CNI plugin code and tests from host to container -RUN mkdir -p $GOPATH/src/github.com/intel/userspace-cni-network-plugin -COPY ./ $GOPATH/src/github.com/intel/userspace-cni-network-plugin/ - -// download dependencies and generate VPP binapi code -WORKDIR $GOPATH/src/github.com/intel/userspace-cni-network-plugin -RUN make install-dep -RUN make install -RUN make diff --git a/docker/unit-tests/Dockerfile.base.packages.in b/docker/unit-tests/Dockerfile.base.packages.in deleted file mode 100644 index 2b29f6ab..00000000 --- a/docker/unit-tests/Dockerfile.base.packages.in +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// install required packages common for CentOS, Fedora and Ubuntu containers -#define PACKAGES \ - findutils \ - gcc \ - git \ - make \ - patch \ - sudo \ - vim \ - wget diff --git a/docker/unit-tests/Dockerfile.centos7.in b/docker/unit-tests/Dockerfile.centos7.in deleted file mode 100644 index 19ca65f8..00000000 --- a/docker/unit-tests/Dockerfile.centos7.in +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "Dockerfile.base.packages.in" - -FROM centos:7 - -RUN yum -y update && \\ - yum -y install PACKAGES && \\ - yum clean all - -#include "Dockerfile.base.in" diff --git a/docker/unit-tests/Dockerfile.centos8.in b/docker/unit-tests/Dockerfile.centos8.in deleted file mode 100644 index 79852d01..00000000 --- a/docker/unit-tests/Dockerfile.centos8.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -FROM centos:8 - -#include "Dockerfile.dnf.in" diff --git a/docker/unit-tests/Dockerfile.dnf.in b/docker/unit-tests/Dockerfile.dnf.in deleted file mode 100644 index ebe25133..00000000 --- a/docker/unit-tests/Dockerfile.dnf.in +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "Dockerfile.base.packages.in" - -RUN dnf -y update && \\ - dnf -y install PACKAGES && \\ - dnf -y clean all - -#include "Dockerfile.base.in" diff --git a/docker/unit-tests/Dockerfile.fedora31.in b/docker/unit-tests/Dockerfile.fedora31.in deleted file mode 100644 index e8b8050e..00000000 --- a/docker/unit-tests/Dockerfile.fedora31.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -FROM fedora:31 - -#include "Dockerfile.dnf.in" diff --git a/docker/unit-tests/Dockerfile.fedora32.in b/docker/unit-tests/Dockerfile.fedora32.in deleted file mode 100644 index e42e6111..00000000 --- a/docker/unit-tests/Dockerfile.fedora32.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -FROM fedora:32 - -#include "Dockerfile.dnf.in" diff --git a/docker/unit-tests/Dockerfile.ubuntu.in b/docker/unit-tests/Dockerfile.ubuntu.in deleted file mode 100644 index 0d43231e..00000000 --- a/docker/unit-tests/Dockerfile.ubuntu.in +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "Dockerfile.base.packages.in" - -RUN apt-get -y update && \\ - apt-get -y upgrade && \\ - apt-get -y install PACKAGES && \\ - apt-get -y clean - -#include "Dockerfile.base.in" diff --git a/docker/unit-tests/Dockerfile.ubuntu16.04.in b/docker/unit-tests/Dockerfile.ubuntu16.04.in deleted file mode 100644 index a34e14cb..00000000 --- a/docker/unit-tests/Dockerfile.ubuntu16.04.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -FROM ubuntu:16.04 - -#include "Dockerfile.ubuntu.in" diff --git a/docker/unit-tests/Dockerfile.ubuntu18.04.in b/docker/unit-tests/Dockerfile.ubuntu18.04.in deleted file mode 100644 index 392ded36..00000000 --- a/docker/unit-tests/Dockerfile.ubuntu18.04.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -FROM ubuntu:18.04 - -#include "Dockerfile.ubuntu.in" diff --git a/docker/unit-tests/Dockerfile.ubuntu20.04.in b/docker/unit-tests/Dockerfile.ubuntu20.04.in deleted file mode 100644 index 217f33c3..00000000 --- a/docker/unit-tests/Dockerfile.ubuntu20.04.in +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -FROM ubuntu:20.04 - -#include "Dockerfile.ubuntu.in" diff --git a/docker/unit-tests/README.md b/docker/unit-tests/README.md deleted file mode 100644 index 2fbbba7e..00000000 --- a/docker/unit-tests/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Dockerfiles for unit test containers - -This directory contains Dockerfiles of container images suitable for execution -of unit tests. Final Dockerfiles are generated by C preprocessor from source -files to enable more flexible and modular Dockerfile definition. - -Please see documentation in the project root directory for details about -unit testing of userspace CNI networking plugin. diff --git a/docker/usrsp-app/usrsp-app.go b/docker/usrsp-app/usrsp-app.go deleted file mode 100644 index 9d9864e1..00000000 --- a/docker/usrsp-app/usrsp-app.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2018-2020 Red Hat, Intel Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// -// VPP implementation of the UserSpace CNI on the host will write -// configuration data in the from of json data to local files. The -// directory containing the files is then mapped to a container. -// -// This application is designed to run in a container, process the -// files written by the host and config the local vSwitch instance in -// the container. -// - -package main - -import ( - "fmt" - "time" - - "k8s.io/client-go/kubernetes" - - "github.com/intel/userspace-cni-network-plugin/cnivpp" - "github.com/intel/userspace-cni-network-plugin/logging" - "github.com/intel/userspace-cni-network-plugin/pkg/configdata" -) - -// Constants -const ( - dbgApp = true -) - -// -// Types -// - -// -// API Functions -// - -// Utility Functions -const DefaultAnnotationsFile = "/etc/podinfo/annotations" - -func cniContainerConfig() (bool, string, error) { - var engine string - var kubeClient kubernetes.Interface - var found bool - - vpp := cnivpp.CniVpp{} - //ovs := cniovs.CniOvs{} - - ifaceList, sharedDir, err := configdata.GetRemoteConfig(DefaultAnnotationsFile) - if err != nil || len(ifaceList) == 0 { - return found, engine, err - } else { - found = true - } - - for i := range ifaceList { - if ifaceList[i].NetConf.LogFile != "" { - logging.SetLogFile(ifaceList[i].NetConf.LogFile) - } - if ifaceList[i].NetConf.LogLevel != "" { - logging.SetLogLevel(ifaceList[i].NetConf.LogLevel) - } - - logging.Debugf("USRSP_APP: iface %v - Data %v", i, ifaceList[i]) - - // Add the requested interface and network - engine = ifaceList[i].NetConf.HostConf.Engine - if ifaceList[i].NetConf.HostConf.Engine == "vpp" { - err = vpp.AddOnHost(&ifaceList[i].NetConf, &ifaceList[i].Args, kubeClient, sharedDir, &ifaceList[i].IPResult) - } else if ifaceList[i].NetConf.HostConf.Engine == "ovs-dpdk" { - //err = ovs.AddOnHost(&ifaceList[i].NetConf, &ifaceList[i].Args, kubeClient, sharedDir, &ifaceList[i].IPResult) - logging.Debugf("USRSP_APP: \"ovs-dpdk\" - Currently nothing TO DO!") - } else { - err = fmt.Errorf("ERROR: Unknown Host Engine:" + ifaceList[i].NetConf.HostConf.Engine) - } - if err != nil { - _ = logging.Errorf("USRSP_APP: %v", err) - } - } - - return found, engine, err -} - -// main -func main() { - var count int = 0 - var engine string - var err error - var found bool - - // Give VPP time to come up - // TBD - Look for /run/vpp-api.sock to exist or something like that - time.Sleep(5 * time.Second) - - for { - count++ - - found, engine, err = cniContainerConfig() - - if dbgApp { - if err != nil { - fmt.Println("ERROR returned:", err) - } - - fmt.Println("LOOP", count, " - FOUND:", found) - } - - // - // Once files have been found, wait 1 more loop and exit. - // - if found { - fmt.Println("") - fmt.Println("") - fmt.Println("Found Configuration and applied.") - - if engine == "vpp" { - fmt.Println("") - fmt.Println("Useful VPP CLI Commands:") - fmt.Println(" vppctl show interface") - fmt.Println(" vppctl show interface addr") - fmt.Println(" vppctl show mode") - fmt.Println(" vppctl show hardware") - } else if engine == "ovs-dpdk" { - fmt.Println("") - fmt.Println("Useful OvS-DPDK CLI Commands:") - fmt.Println(" ovs-vsctl list open_vswitch") - fmt.Println(" ovs-vsctl list port") - fmt.Println(" ovs-vsctl list bridge") - } - - fmt.Println("") - fmt.Println("DONE: Exiting app - Press to return to prompt") - break - } - - if count > 10 { - break - } - - time.Sleep(3 * time.Second) - } -} diff --git a/docker/vpp-centos-userspace-cni/80-vpp.conf b/docker/vpp-centos-userspace-cni/80-vpp.conf deleted file mode 100644 index 0b942e74..00000000 --- a/docker/vpp-centos-userspace-cni/80-vpp.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Number of 2MB hugepages desired -vm.nr_hugepages=512 - -# Must be greater than or equal to (2 * vm.nr_hugepages). -vm.max_map_count=2048 - -# All groups allowed to access hugepages -vm.hugetlb_shm_group=0 - -# Shared Memory Max must be greator or equal to the total size of hugepages. -# For 2MB pages, TotalHugepageSize = vm.nr_hugepages * 2 * 1024 * 1024 -# If the existing kernel.shmmax setting (cat /sys/proc/kernel/shmmax) -# is greater than the calculated TotalHugepageSize then set this parameter -# to current shmmax value. -kernel.shmmax=1073741824 diff --git a/docker/vpp-centos-userspace-cni/Dockerfile b/docker/vpp-centos-userspace-cni/Dockerfile deleted file mode 100644 index 4d736b63..00000000 --- a/docker/vpp-centos-userspace-cni/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -# -# To build: -# docker build --rm -t vpp-centos-userspace-cni . -# - - -# -------- Builder stage. -FROM centos:8@sha256:a27fd8080b517143cbbbab9dfb7c8571c40d67d534bbdee55bd6c473f432b177 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# Install VPP - Needed by CNI-VPP -RUN curl -s https://packagecloud.io/install/repositories/fdio/release/script.rpm.sh | bash -RUN yum install -y epel-release -RUN yum install -y epel-release vpp-plugins vpp-devel vpp-api-python vpp-api-lua; yum clean all - -# -# Download and Build Container usrsp-app -# - -# Pull in GO -RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo -RUN yum install -y git golang make - -# Build the usrsp-app -WORKDIR /root/go/src/ -RUN go get github.com/intel/userspace-cni-network-plugin > /tmp/UserspaceDockerBuild.log 2>&1 || echo "Can ignore no GO files." -WORKDIR /root/go/src/github.com/intel/userspace-cni-network-plugin -RUN make extras -RUN cp docker/usrsp-app/usrsp-app /usr/sbin/usrsp-app - - -# -------- Import stage. -# Docker 17.05 or higher, remove ## -##FROM centos - -# Install UserSpace CNI -##COPY --from=0 /usr/sbin/usrsp-app /usr/sbin/usrsp-app - - -# Install VPP -##RUN curl -s https://packagecloud.io/install/repositories/fdio/release/script.rpm.sh | bash -##RUN yum install -y epel-release -##RUN yum install -y vpp-plugins vpp-devel vpp-api-python vpp-api-lua; yum clean all - -# Overwrite VPP systemfiles -COPY startup.conf /etc/vpp/startup.conf -COPY 80-vpp.conf /etc/sysctl.d/80-vpp.conf - - -# Install script to start both VPP and usrsp-app -COPY vppcni.sh vppcni.sh - - -# Setup VPP UserGroup and User -#RUN useradd --no-log-init -r -g vpp vpp -#USER vpp - - -# For Development, overwrite repo generated usrsp-app with local development binary. -# Needs to be commented out before each merge. -#COPY usrsp-app /usr/sbin/usrsp-app - - -CMD ["bash", "-C", "./vppcni.sh"] -#CMD [ "./vppcni.sh" ] diff --git a/docker/vpp-centos-userspace-cni/README.md b/docker/vpp-centos-userspace-cni/README.md deleted file mode 100644 index c8eff6f8..00000000 --- a/docker/vpp-centos-userspace-cni/README.md +++ /dev/null @@ -1,195 +0,0 @@ -# DockerHub Image: bmcfall/vpp-centos-userspace-cni -This directory contains the files needed to build the docker image located in: - https://hub.docker.com/r/bmcfall/vpp-centos-userspace-cni/ - -This image is based on CentOS (latest) base image built with VPP 19.04.1 and a -User Space CNI application (usrsp-app). Source code for the usrsp-app is in this -same repo: - https://github.com/intel/userspace-cni-network-plugin - -The User Space CNI inconjunction with the VPP CNI Library (cnivpp) creates -interfaces on the host, like memif interface, adds the host side of the -interface to a local network, then copies information needed in the container -into annotations. The container, like this one, boots up, starts a local instance -of VPP, then runs the usrsp-app to read the annotataions, looking for the needed -data. When found, usrsp-app consumes the data and writes to the local VPP instance -via the VPP GO API. This container then drops into bash for additional testing and -debugging. - - -# Build Instructions for vpp-centos-userspace-cni Docker Image -Get the **user-space-net-plugin** repo: -``` - cd $GOPATH/src/ - go get github.com/intel/userspace-cni-network-plugin -``` - -Build the docker image: -``` - cd $GOPATH/src/github.com/intel/userspace-cni-network-plugin/docker/vpp-centos-userspace-cni/ - docker build --rm -t vpp-centos-userspace-cni . -``` - - -## Development Image -The above process pulls the **usrsp-app** from the upstream source. If there are -local changes that need to be tested, then build **user-space-net-plugin** to -get the **usrsp-app** binary and copy the **usrsp-app** into the image directory: -``` - cd $GOPATH/src/github.com/intel/userspace-cni-network-plugin/ - make extras - cp docker/usrsp-app/usrsp-app docker/vpp-centos-userspace-cni/. -``` - -Update the **Dockerfile** to use the local binary by uncommenting the COPY -command. **Make sure not to check this change in.** -``` - vi docker/vpp-centos-userspace-cni/Dockerfile - : - - # For Development, overwrite repo generated usrsp-app with local development binary. - # Needs to be commented out before each merge. - COPY usrsp-app /usr/sbin/usrsp-app <-- Uncomment -``` - -Depending on the state of the **usrsp-app** and the number of changes, the -**usrsp-app** may not build from upstream, so building of **usrsp-app** in -the container image may also need to be commented out. **Make sure not to -check this change in.** -``` -# Build the usrsp-app -WORKDIR /root/go/src/github.com/intel/ -RUN git clone https://github.com/intel/userspace-cni-network-plugin -WORKDIR /root/go/src/github.com/intel/userspace-cni-network-plugin -#RUN make extras <-- Comment out -#RUN cp docker/usrsp-app/usrsp-app /usr/sbin/usrsp-app <-- Comment out -``` - -Build the docker image as described above. - -# To run -The following directory contains some sample yaml files that create -two networks and creates two pods with two addtional interfaces (one -for each network). Once created, the pods can ping each other over -these two networks. -'github.com/intel/userspace-cni-network-plugin/examples/vpp-memif-ping/' - -Example: -Currently, Userspace CNI does not manage networks between nodes. For testing, -**userspace-vpp-pod-1.yaml** and **userspace-vpp-pod-2.yaml** have the -**nodeSelector** of **vswitch: vpp** included so both pods end up on the same -node. Either remove this **nodeSelector** and ensure the VPP instaneces on -different nodes can pass traffic between nodes on the bridges defined in -**userspace-vpp-netAttach-1.yaml** and **userspace-vpp-netAttach-2.yaml**, -or add a label as follows, where **** is a valid node from -**kubectl get nodes**: -``` -kubectl label nodes vswitch=vpp -``` - -Create the two networks and then the two pods: -``` -kubectl create -f examples/vpp-memif-ping/userspace-vpp-netAttach-1.yaml -kubectl create -f examples/vpp-memif-ping/userspace-vpp-netAttach-2.yaml -kubectl create -f examples/vpp-memif-ping/userspace-vpp-pod-1.yaml -kubectl create -f examples/vpp-memif-ping/userspace-vpp-pod-2.yaml -``` - -Log into the two pods and retrieve the IP addresses, and run a ping between them: -``` -kubectl exec -it userspace-vpp-pod-1 -- sh -sh-4.2# vppctl show interface addr -local0 (dn): -memif1/0 (up): - L3 10.56.217.161/24 -memif2/0 (up): - L3 10.77.217.161/24 -sh-4.2# -``` -``` -kubectl exec -it userspace-vpp-pod-2 -- sh -sh-4.2# vppctl show interface addr -local0 (dn): -memif1/0 (up): - L3 10.56.217.162/24 -memif2/0 (up): - L3 10.77.217.162/24 -sh-4.2# vppctl ping 10.56.217.161 -116 bytes from 10.56.217.161: icmp_seq=2 ttl=64 time=79.4484 ms -116 bytes from 10.56.217.161: icmp_seq=3 ttl=64 time=127.9922 ms -116 bytes from 10.56.217.161: icmp_seq=4 ttl=64 time=82.9510 ms -116 bytes from 10.56.217.161: icmp_seq=5 ttl=64 time=77.0007 ms - -Statistics: 5 sent, 4 received, 20% packet loss -sh-4.2# vppctl ping 10.77.217.161 -116 bytes from 10.77.217.161: icmp_seq=2 ttl=64 time=148.0164 ms -116 bytes from 10.77.217.161: icmp_seq=3 ttl=64 time=71.9887 ms -116 bytes from 10.77.217.161: icmp_seq=4 ttl=64 time=151.0084 ms -116 bytes from 10.77.217.161: icmp_seq=5 ttl=64 time=73.5451 ms - -Statistics: 5 sent, 4 received, 20% packet loss -sh-4.2# -``` - -# vpp-centos-userspace-cni Docker Image Nuances -Below are a couple points about the image that will probably need to change: - -## VPP Version -This image is based on VPP 19.04.1, which is taken from the upstream -https://packagecloud.io/fdio/ repository. - -**NOTE:** Care must be taken to ensure the same version of VPP is used to build -the cnivpp library, the usrsp-app, the Docker Image and running on the host. -Otherwise there may be an API version mismatch. - -## Volumes and Devices -Inside the sample yaml files the pod is started with the following volume mounts: -``` -vi github.com/intel/userspace-cni-network-plugin/examples/vpp-memif-ping/userspace-vpp-pod-1.yaml -: - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage -: - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /var/lib/cni/usrspcni/023bcd123/ - - name: hugepage - emptyDir: - medium: HugePages -``` -Where: -* **shared-dir** maps **/var/lib/cni/usrspcni/023bcd123** on host to -**/var/lib/cni/usrspcni** in the container. - * This directory contains any files, like the virtio socket files, shared -between the host and the container. - * The **023bcd123** sub-directory is intended to a be a unique folder -per pod on the host. - * Work is in progress for an Addmission Controller to automatically insert -this volume mount into the Pod Spec and generate a random sub-directory. -* **podinfo** is a Downward API definition which maps **/etc/podinfo** into the -container. - * All labels are inserted into a file named **labels** into this directory by -kubernetes. - * All annotations are inserted into a file named **annotations** into this directory -by kubernetes. - * Work is in progress for an Addmission Controller to automatically insert -this Downward API mount into the Pod Spec. -* **hugepage** maps **/dev/hugepages** on the host to **/dev/hugepages** in the -container. - * Mapping hugepages into the Container, needed by DPDK application. diff --git a/docker/vpp-centos-userspace-cni/vppcni.sh b/docker/vpp-centos-userspace-cni/vppcni.sh deleted file mode 100755 index 8e5658ab..00000000 --- a/docker/vpp-centos-userspace-cni/vppcni.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Start the first process -/usr/bin/vpp -c /etc/vpp/startup.conf &> vppBoot.log & -status=$? -if [ $status -ne 0 ]; then - echo "Failed to start VPP: $status" - exit $status -fi - -sleep 2 - -# Start the second process -/usr/sbin/usrsp-app & -status=$? -if [ $status -ne 0 ]; then - echo "Failed to start usrsp-app for VPP: $status" - exit $status -fi - -while : -do - sleep 1000 -done - diff --git a/examples/crd-userspace-net-ovs-no-ipam.yaml b/examples/crd-userspace-net-ovs-no-ipam.yaml deleted file mode 100644 index b9fe959a..00000000 --- a/examples/crd-userspace-net-ovs-no-ipam.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: userspace-networkobj -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "userspace", - "host": { - "engine": "ovs-dpdk", - "iftype": "vhostuser" - } - }' diff --git a/examples/ovs-vhost/testpmd-pod-1.yaml b/examples/ovs-vhost/testpmd-pod-1.yaml new file mode 100644 index 00000000..7e18f2b9 --- /dev/null +++ b/examples/ovs-vhost/testpmd-pod-1.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: ovs-app1 + labels: + name: ovs-app1 + namespace: ovs + annotations: + k8s.v1.cni.cncf.io/networks: userspace-ovs-net-1 + userspace/mappedDir: /var/lib/cni/usrspcni/ +spec: + nodeSelector: + kubernetes.io/hostname: kind-control-plane # replace this with your hostname + hostname: ovs-app1 + subdomain: ovs + containers: + - image: localhost:5000/testpmd #replace this with your docker reg and image name + imagePullPolicy: IfNotPresent + name: ovs-app1 + volumeMounts: + - name: podinfo + mountPath: /etc/podinfo + - name: hugepage + mountPath: /hugepages + - name: shared-dir + mountPath: /var/lib/cni/usrspcni/ + - name: vfio + mountPath: /dev/vfio/ + resources: + requests: + hugepages-2Mi: 1Gi + memory: "500Mi" + cpu: "5" + limits: + hugepages-2Mi: 1Gi + memory: "500Mi" + cpu: "5" +# command: ["/bin/sh"] +# args: ["-c", "sleep inf"] + restartPolicy: Always + volumes: + - name: podinfo + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations + - name: hugepage + emptyDir: + medium: HugePages + - name: shared-dir + hostPath: + path: /run/openvswitch/app1 + - name: vfio + hostPath: + path: /dev/vfio/ diff --git a/examples/ovs-vhost/testpmd-pod-2.yaml b/examples/ovs-vhost/testpmd-pod-2.yaml new file mode 100644 index 00000000..392bc3b2 --- /dev/null +++ b/examples/ovs-vhost/testpmd-pod-2.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: ovs-app2 + labels: + name: ovs-app2 + namespace: ovs + annotations: + k8s.v1.cni.cncf.io/networks: userspace-ovs-net-1 + userspace/mappedDir: /var/lib/cni/usrspcni/ +spec: + nodeSelector: + kubernetes.io/hostname: kind-control-plane # replace this with your hostname + hostname: ovs-app2 + subdomain: ovs + containers: + - image: localhost:5000/testpmd #replace this with your docker reg and image name + imagePullPolicy: IfNotPresent + name: ovs-app1 + volumeMounts: + - name: podinfo + mountPath: /etc/podinfo + - name: hugepage + mountPath: /hugepages + - name: shared-dir + mountPath: /var/lib/cni/usrspcni/ + - name: vfio + mountPath: /dev/vfio/ + resources: + requests: + hugepages-2Mi: 1Gi + memory: "500Mi" + cpu: "5" + limits: + hugepages-2Mi: 1Gi + memory: "500Mi" + cpu: "5" +# command: ["/bin/sh"] +# args: ["-c", "sleep inf"] + restartPolicy: Always + volumes: + - name: podinfo + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations + - name: hugepage + emptyDir: + medium: HugePages + - name: shared-dir + hostPath: + path: /run/openvswitch/app2 + - name: vfio + hostPath: + path: /dev/vfio/ diff --git a/examples/ovs-vhost/userspace-ovs-netAttach-1.yaml b/examples/ovs-vhost/userspace-ovs-netAttach-1.yaml index 75e3df99..f3dc623c 100644 --- a/examples/ovs-vhost/userspace-ovs-netAttach-1.yaml +++ b/examples/ovs-vhost/userspace-ovs-netAttach-1.yaml @@ -2,6 +2,7 @@ apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: name: userspace-ovs-net-1 + namespace: ovs spec: config: '{ "cniVersion": "0.3.1", @@ -15,7 +16,7 @@ spec: "iftype": "vhostuser", "netType": "bridge", "vhost": { - "mode": "server" + "mode": "client" }, "bridge": { "bridgeName": "br-4" @@ -26,7 +27,7 @@ spec: "iftype": "vhostuser", "netType": "interface", "vhost": { - "mode": "client" + "mode": "server" } }, "ipam": { diff --git a/examples/ovs-vhost/userspace-ovs-netAttach-2.yaml b/examples/ovs-vhost/userspace-ovs-netAttach-2.yaml deleted file mode 100644 index 74c93bb8..00000000 --- a/examples/ovs-vhost/userspace-ovs-netAttach-2.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: userspace-ovs-net-2 -spec: - config: '{ - "cniVersion": "0.3.1", - "type": "userspace", - "name": "userspace-ovs-net-2", - "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig", - "logFile": "/var/log/userspace-ovs-net-2-cni.log", - "logLevel": "debug", - "host": { - "engine": "ovs-dpdk", - "iftype": "vhostuser", - "netType": "bridge", - "vhost": { - "mode": "server" - }, - "bridge": { - "bridgeName": "br-5" - } - }, - "container": { - "engine": "ovs-dpdk", - "iftype": "vhostuser", - "netType": "interface", - "vhost": { - "mode": "client" - } - }, - "ipam": { - "type": "host-local", - "subnet": "10.77.217.0/24", - "rangeStart": "10.77.217.131", - "rangeEnd": "10.77.217.190", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ], - "gateway": "10.77.217.1" - } - }' diff --git a/examples/ovs-vhost/userspace-ovs-pod-1.yaml b/examples/ovs-vhost/userspace-ovs-pod-1.yaml deleted file mode 100644 index 4bd3d184..00000000 --- a/examples/ovs-vhost/userspace-ovs-pod-1.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: userspace-ovs-pod-1 - annotations: - k8s.v1.cni.cncf.io/networks: userspace-ovs-net-1, userspace-ovs-net-2 -spec: - containers: - - name: multi-vhost-example - image: dpdk-app-centos:latest - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 1Gi - limits: - hugepages-1Gi: 1Gi - nodeSelector: - vswitch: ovs - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /usr/local/var/run/openvswitch/023bcd123/ - - name: hugepage - emptyDir: - medium: HugePages diff --git a/examples/ovs-vhost/userspace-ovs-pod-2.yaml b/examples/ovs-vhost/userspace-ovs-pod-2.yaml deleted file mode 100644 index a9ccaa64..00000000 --- a/examples/ovs-vhost/userspace-ovs-pod-2.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: userspace-ovs-pod-2 - annotations: - k8s.v1.cni.cncf.io/networks: userspace-ovs-net-1, userspace-ovs-net-2 -spec: - containers: - - name: multi-vhost-example - image: dpdk-app-centos:latest - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 1Gi - limits: - hugepages-1Gi: 1Gi - nodeSelector: - vswitch: ovs - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /usr/local/var/run/openvswitch/4ed69c30/ - - name: hugepage - emptyDir: - medium: HugePages diff --git a/examples/pod-multi-vhost.yaml b/examples/pod-multi-vhost.yaml deleted file mode 100644 index 4ea71f79..00000000 --- a/examples/pod-multi-vhost.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: multi-vhost-example - annotations: - k8s.v1.cni.cncf.io/networks: userspace-networkobj, userspace-networkobj -spec: - containers: - - name: multi-vhost-example - image: ubuntu-dpdk - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /vhu/ - name: socket - - mountPath: /mnt/huge - name: hugepage - resources: - requests: - memory: 1Gi - limits: - hugepages-1Gi: 1Gi - command: ["sleep", "infinity"] - volumes: - - name: socket - hostPath: - path: /var/lib/cni/vhostuser/ - - name: hugepage - emptyDir: - medium: HugePages diff --git a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-netAttach-2.yaml b/examples/vpp-memif-ping/kubeConfig/userspace-vpp-netAttach-2.yaml deleted file mode 100644 index a4464d80..00000000 --- a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-netAttach-2.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: userspace-vpp-net-2 -spec: - config: '{ - "cniVersion": "0.3.1", - "type": "userspace", - "name": "userspace-vpp-net-2", - "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig", - "logFile": "/var/log/userspace-vpp-net-2-cni.log", - "logLevel": "debug", - "host": { - "engine": "vpp", - "iftype": "memif", - "netType": "bridge", - "memif": { - "role": "master", - "mode": "ethernet" - }, - "bridge": { - "bridgeName": "12" - } - }, - "container": { - "engine": "vpp", - "iftype": "memif", - "netType": "interface", - "memif": { - "role": "slave", - "mode": "ethernet" - } - }, - "ipam": { - "type": "host-local", - "subnet": "10.77.217.0/24", - "rangeStart": "10.77.217.131", - "rangeEnd": "10.77.217.190", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ], - "gateway": "10.77.217.1" - } - }' - diff --git a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-pod-1.yaml b/examples/vpp-memif-ping/kubeConfig/userspace-vpp-pod-1.yaml deleted file mode 100644 index b1edf35c..00000000 --- a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-pod-1.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: userspace-vpp-pod-1 - annotations: - k8s.v1.cni.cncf.io/networks: userspace-vpp-net-1, userspace-vpp-net-2 -spec: - containers: - - name: multi-memif-example - image: dpdk-app-centos:latest - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 2Mi - limits: - hugepages-2Mi: 1024Mi - nodeSelector: - vswitch: vpp - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /var/run/vpp/023bcd123/ - - name: hugepage - emptyDir: - medium: HugePages - diff --git a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-pod-2.yaml b/examples/vpp-memif-ping/kubeConfig/userspace-vpp-pod-2.yaml deleted file mode 100644 index 72825472..00000000 --- a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-pod-2.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: userspace-vpp-pod-2 - annotations: - k8s.v1.cni.cncf.io/networks: userspace-vpp-net-1, userspace-vpp-net-2 -spec: - containers: - - name: multi-memif-example - image: dpdk-app-centos:latest - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 2Mi - limits: - hugepages-2Mi: 1024Mi - nodeSelector: - vswitch: vpp - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /var/run/vpp/4ed69c30/ - - name: hugepage - emptyDir: - medium: HugePages - diff --git a/examples/vpp-memif-ping/sharedDir/userspace-vpp-netAttach-1.yaml b/examples/vpp-memif-ping/sharedDir/userspace-vpp-netAttach-1.yaml deleted file mode 100644 index 25947859..00000000 --- a/examples/vpp-memif-ping/sharedDir/userspace-vpp-netAttach-1.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: userspace-vpp-net-1 -spec: - config: '{ - "cniVersion": "0.3.1", - "type": "userspace", - "name": "userspace-vpp-net-1", - "sharedDir": "/var/run/vpp/", - "logFile": "/var/log/userspace-vpp-net-1-cni.log", - "logLevel": "debug", - "host": { - "engine": "vpp", - "iftype": "memif", - "netType": "bridge", - "memif": { - "role": "master", - "mode": "ethernet" - }, - "bridge": { - "bridgeName": "4" - } - }, - "container": { - "engine": "vpp", - "iftype": "memif", - "netType": "interface", - "memif": { - "role": "slave", - "mode": "ethernet" - } - }, - "ipam": { - "type": "host-local", - "subnet": "10.56.217.0/24", - "rangeStart": "10.56.217.131", - "rangeEnd": "10.56.217.190", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ], - "gateway": "10.56.217.1" - } - }' - diff --git a/examples/vpp-memif-ping/sharedDir/userspace-vpp-netAttach-2.yaml b/examples/vpp-memif-ping/sharedDir/userspace-vpp-netAttach-2.yaml deleted file mode 100644 index dd05ef9d..00000000 --- a/examples/vpp-memif-ping/sharedDir/userspace-vpp-netAttach-2.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: userspace-vpp-net-2 -spec: - config: '{ - "cniVersion": "0.3.1", - "type": "userspace", - "name": "userspace-vpp-net-2", - "sharedDir": "/var/run/vpp", - "logFile": "/var/log/userspace-vpp-net-2-cni.log", - "logLevel": "debug", - "host": { - "engine": "vpp", - "iftype": "memif", - "netType": "bridge", - "memif": { - "role": "master", - "mode": "ethernet" - }, - "bridge": { - "bridgeName": "12" - } - }, - "container": { - "engine": "vpp", - "iftype": "memif", - "netType": "interface", - "memif": { - "role": "slave", - "mode": "ethernet" - } - }, - "ipam": { - "type": "host-local", - "subnet": "10.77.217.0/24", - "rangeStart": "10.77.217.131", - "rangeEnd": "10.77.217.190", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ], - "gateway": "10.77.217.1" - } - }' - diff --git a/examples/vpp-memif-ping/sharedDir/userspace-vpp-pod-1.yaml b/examples/vpp-memif-ping/sharedDir/userspace-vpp-pod-1.yaml deleted file mode 100644 index 1506b8bc..00000000 --- a/examples/vpp-memif-ping/sharedDir/userspace-vpp-pod-1.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: userspace-vpp-pod-1 - annotations: - k8s.v1.cni.cncf.io/networks: userspace-vpp-net-1, userspace-vpp-net-2 - userspace/mappedDir: /var/lib/cni/usrspcni/ -spec: - containers: - - name: multi-memif-example - image: dpdk-app-centos:latest - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 2Mi - limits: - hugepages-2Mi: 1024Mi - nodeSelector: - vswitch: vpp - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /var/run/vpp/ - - name: hugepage - emptyDir: - medium: HugePages - diff --git a/examples/vpp-memif-ping/sharedDir/userspace-vpp-pod-2.yaml b/examples/vpp-memif-ping/sharedDir/userspace-vpp-pod-2.yaml deleted file mode 100644 index 3f3978a7..00000000 --- a/examples/vpp-memif-ping/sharedDir/userspace-vpp-pod-2.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: userspace-vpp-pod-2 - annotations: - k8s.v1.cni.cncf.io/networks: userspace-vpp-net-1, userspace-vpp-net-2 - userspace/mappedDir: /var/lib/cni/usrspcni/ -spec: - containers: - - name: multi-memif-example - image: dpdk-app-centos:latest - imagePullPolicy: Never - securityContext: - privileged: true - volumeMounts: - - mountPath: /etc/podinfo - name: podinfo - readOnly: false - - mountPath: /var/lib/cni/usrspcni/ - name: shared-dir - - mountPath: /dev/hugepages - name: hugepage - resources: - requests: - memory: 2Mi - limits: - hugepages-2Mi: 1024Mi - nodeSelector: - vswitch: vpp - volumes: - - name: podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - - name: shared-dir - hostPath: - path: /var/run/vpp/ - - name: hugepage - emptyDir: - medium: HugePages - diff --git a/examples/vpp-memif-ping/startup.conf b/examples/vpp-memif-ping/startup.conf new file mode 100644 index 00000000..d986c25e --- /dev/null +++ b/examples/vpp-memif-ping/startup.conf @@ -0,0 +1,152 @@ + +unix { + nodaemon + log /var/log/vpp/vpp-app.log + full-coredump + cli-listen /run/vpp/cli1.sock + gid vpp +} + +api-trace { +## This stanza controls binary API tracing. Unless there is a very strong reason, +## please leave this feature enabled. + on +## Additional parameters: +## +## To set the number of binary API trace records in the circular buffer, configure nitems +## +## nitems +## +## To save the api message table decode tables, configure a filename. Results in /tmp/ +## Very handy for understanding api message changes between versions, identifying missing +## plugins, and so forth. +## +## save-api-table +} + +api-segment { + gid vpp +} + +cpu { + ## In the VPP there is one main thread and optionally the user can create worker(s) + ## The main thread and worker thread(s) can be pinned to CPU core(s) manually or automatically + + ## Manual pinning of thread(s) to CPU core(s) + + ## Set logical CPU core where main thread runs + # main-core 1 + + ## Set logical CPU core(s) where worker threads are running + # corelist-workers 2-3,18-19 + + ## Automatic pinning of thread(s) to CPU core(s) + + ## Sets number of CPU core(s) to be skipped (1 ... N-1) + ## Skipped CPU core(s) are not used for pinning main thread and working thread(s). + ## The main thread is automatically pinned to the first available CPU core and worker(s) + ## are pinned to next free CPU core(s) after core assigned to main thread + # skip-cores 4 + + ## Specify a number of workers to be created + ## Workers are pinned to N consecutive CPU cores while skipping "skip-cores" CPU core(s) + ## and main thread's CPU core + # workers 2 + + ## Set scheduling policy and priority of main and worker threads + + ## Scheduling policy options are: other (SCHED_OTHER), batch (SCHED_BATCH) + ## idle (SCHED_IDLE), fifo (SCHED_FIFO), rr (SCHED_RR) + # scheduler-policy fifo + + ## Scheduling priority is used only for "real-time policies (fifo and rr), + ## and has to be in the range of priorities supported for a particular policy + # scheduler-priority 50 +} + +dpdk { + no-pci + + ## Change default settings for all intefaces + # dev default { + ## Number of receive queues, enables RSS + ## Default is 1 + # num-rx-queues 3 + + ## Number of transmit queues, Default is equal + ## to number of worker threads or 1 if no workers treads + # num-tx-queues 3 + + ## Number of descriptors in transmit and receive rings + ## increasing or reducing number can impact performance + ## Default is 1024 for both rx and tx + # num-rx-desc 512 + # num-tx-desc 512 + + ## VLAN strip offload mode for interface + ## Default is off + # vlan-strip-offload on + # } + + ## Allowlist specific interface by specifying PCI address + # dev 0000:02:00.0 + + ## Allowlist specific interface by specifying PCI address and in + ## addition specify custom parameters for this interface + # dev 0000:02:00.1 { + # num-rx-queues 2 + # } + + ## Specify bonded interface and its slaves via PCI addresses + ## + ## Bonded interface in XOR load balance mode (mode 2) with L3 and L4 headers + # vdev eth_bond0,mode=2,slave=0000:02:00.0,slave=0000:03:00.0,xmit_policy=l34 + # vdev eth_bond1,mode=2,slave=0000:02:00.1,slave=0000:03:00.1,xmit_policy=l34 + ## + ## Bonded interface in Active-Back up mode (mode 1) + # vdev eth_bond0,mode=1,slave=0000:02:00.0,slave=0000:03:00.0 + # vdev eth_bond1,mode=1,slave=0000:02:00.1,slave=0000:03:00.1 + + ## Change UIO driver used by VPP, Options are: igb_uio, vfio-pci, + ## uio_pci_generic or auto (default) + # uio-driver vfio-pci + + ## Disable mutli-segment buffers, improves performance but + ## disables Jumbo MTU support + # no-multi-seg + + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per CPU socket. + ## Default is 16384 + # num-mbufs 128000 + + ## Change hugepages allocation per-socket, needed only if there is need for + ## larger number of mbufs. Default is 256M on each detected CPU socket + # socket-mem 2048,2048 + + ## Disables UDP / TCP TX checksum offload. Typically needed for use + ## faster vector PMDs (together with no-multi-seg) + # no-tx-checksum-offload +} + + +# plugins { + ## Adjusting the plugin path depending on where the VPP plugins are + # path /home/bms/vpp/build-root/install-vpp-native/vpp/lib64/vpp_plugins + + ## Disable all plugins by default and then selectively enable specific plugins + # plugin default { disable } + # plugin dpdk_plugin.so { enable } + # plugin acl_plugin.so { enable } + + ## Enable all plugins by default and then selectively disable specific plugins + # plugin dpdk_plugin.so { disable } + # plugin acl_plugin.so { disable } +# } + + ## Alternate syntax to choose plugin path + # plugin_path /home/bms/vpp/build-root/install-vpp-native/vpp/lib64/vpp_plugins + +socksvr { + socket-name /run/vpp/api1.sock +} diff --git a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-netAttach-1.yaml b/examples/vpp-memif-ping/userspace-vpp-netAttach.yaml similarity index 98% rename from examples/vpp-memif-ping/kubeConfig/userspace-vpp-netAttach-1.yaml rename to examples/vpp-memif-ping/userspace-vpp-netAttach.yaml index 8dfb6533..54b357dc 100644 --- a/examples/vpp-memif-ping/kubeConfig/userspace-vpp-netAttach-1.yaml +++ b/examples/vpp-memif-ping/userspace-vpp-netAttach.yaml @@ -1,7 +1,9 @@ +--- apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: name: userspace-vpp-net-1 + namespace: vpp spec: config: '{ "cniVersion": "0.3.1", @@ -44,4 +46,3 @@ spec: "gateway": "10.56.217.1" } }' - diff --git a/examples/vpp-memif-ping/vpp-app-pod-1.yaml b/examples/vpp-memif-ping/vpp-app-pod-1.yaml new file mode 100644 index 00000000..680f99ac --- /dev/null +++ b/examples/vpp-memif-ping/vpp-app-pod-1.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: vpp-app1 + labels: + name: vpp-app1 + namespace: vpp + annotations: + k8s.v1.cni.cncf.io/networks: userspace-vpp-net-1 + userspace/mappedDir: /var/lib/cni/usrspcni/ +spec: + nodeSelector: + kubernetes.io/hostname: kind-control-plane #replace this with your hostname + hostname: vpp-app1 + subdomain: vpp + containers: + - image: ligato/vpp-base:23.02 #imagename + imagePullPolicy: IfNotPresent + name: vpp-app1 + volumeMounts: + - name: podinfo + mountPath: /etc/podinfo + - name: vpp-startup-config + mountPath: /etc/vpp/ + - name: hugepage + mountPath: /hugepages + - name: shared-dir + mountPath: /var/lib/cni/usrspcni/ + - name: scripts + mountPath: /vpp/ + resources: + requests: + hugepages-2Mi: 1Gi + memory: "1Gi" + cpu: "3" + limits: + hugepages-2Mi: 1Gi + memory: "1Gi" + cpu: "3" + restartPolicy: Always + volumes: + - name: podinfo + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations + - name: vpp-startup-config + configMap: + name: vpp-startup-config + - name: hugepage + emptyDir: + medium: HugePages + - name: shared-dir + hostPath: + path: /run/vpp/app1 + - name: scripts + configMap: + name: vpp-pod-setup-memif + defaultMode: 0777 diff --git a/examples/vpp-memif-ping/vpp-app-pod-2.yaml b/examples/vpp-memif-ping/vpp-app-pod-2.yaml new file mode 100644 index 00000000..539a19bc --- /dev/null +++ b/examples/vpp-memif-ping/vpp-app-pod-2.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: vpp-app2 + labels: + name: vpp-app2 + namespace: vpp + annotations: + k8s.v1.cni.cncf.io/networks: userspace-vpp-net-1 + userspace/mappedDir: /var/lib/cni/usrspcni/ +spec: + nodeSelector: + kubernetes.io/hostname: kind-control-plane # replace this with your hostname + hostname: vpp-app2 + subdomain: vpp + containers: + - image: ligato/vpp-base:23.02 #imagename + imagePullPolicy: IfNotPresent + name: vpp-app1 + volumeMounts: + - name: podinfo + mountPath: /etc/podinfo + - name: vpp-startup-config + mountPath: /etc/vpp/ + - name: hugepage + mountPath: /hugepages + - name: shared-dir + mountPath: /var/lib/cni/usrspcni/ + - name: scripts + mountPath: /vpp/ + resources: + requests: + hugepages-2Mi: 1Gi + memory: "1Gi" + cpu: "3" + limits: + hugepages-2Mi: 1Gi + memory: "1Gi" + cpu: "3" + restartPolicy: Always + volumes: + - name: podinfo + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations + - name: vpp-startup-config + configMap: + name: vpp-startup-config + - name: hugepage + emptyDir: + medium: HugePages + - name: shared-dir + hostPath: + path: /run/vpp/app2 + - name: scripts + configMap: + name: vpp-pod-setup-memif + defaultMode: 0777 diff --git a/examples/vpp-memif-ping/vpp-pod-setup-memif.sh b/examples/vpp-memif-ping/vpp-pod-setup-memif.sh new file mode 100755 index 00000000..89ae8c1a --- /dev/null +++ b/examples/vpp-memif-ping/vpp-pod-setup-memif.sh @@ -0,0 +1,31 @@ +#!/bin/bash +socket=$(ls /var/lib/cni/usrspcni/) + +containerid=$(grep containerID /etc/podinfo/annotations |cut -d = -f 2 | sed 's/"//g') +echo "${containerid:0:12}" + + + +echo "sh int addr" +vppctl "sh int addr" +echo "create memif socket id 1 filename /var/lib/cni/usrspcni/$socket" +vppctl "create memif socket id 1 filename /var/lib/cni/usrspcni/$socket" +echo "create interface memif id 0 socket-id 1 slave no-zero-copy" +vppctl "create interface memif id 0 socket-id 1 slave no-zero-copy" +echo "set int state memif1/0 up" +vppctl "set int state memif1/0 up" +if hostname | grep -q app1; then + echo app1 + echo "set int ip address memif1/0 192.168.1.3/24" + vppctl "set int ip address memif1/0 192.168.1.3/24" +else + echo app2 + echo "set int ip address memif1/0 192.168.1.4/24" + vppctl "set int ip address memif1/0 192.168.1.4/24" + +fi +echo "sh int addr" +vppctl "sh int addr" +echo "sh memif" +vppctl "sh memif" +exit 0 diff --git a/scripts/.usrsp-docker-run.sh.swp b/scripts/.usrsp-docker-run.sh.swp deleted file mode 100644 index 82083392..00000000 Binary files a/scripts/.usrsp-docker-run.sh.swp and /dev/null differ