Add multiarch support

Signed-off-by: Patrik Cyvoct <[email protected]>

Author: Sh4d1

.gitignore

@@ -1,7 +1,8 @@
-/bin/
+/bin/*
 /Dockerfile
 /test/bin/
 /*.tar.gz
 ci.env
 pr.env
 junit*.xml
+qemu*

Dockerfile.in

@@ -15,15 +15,15 @@
 FROM @BASEIMAGE@
 MAINTAINER Random Liu <[email protected]>
 
+CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
 RUN clean-install util-linux libsystemd0 bash
 
 # Avoid symlink of /etc/localtime.
 RUN test -h /etc/localtime && rm -f /etc/localtime && cp /usr/share/zoneinfo/UTC /etc/localtime || true
 
-ADD ./bin/node-problem-detector /node-problem-detector
+ADD ./bin/node-problem-detector-BINARCH /node-problem-detector
 
 # Below command depends on ENABLE_JOURNAL=1.
-ADD ./bin/log-counter /home/kubernetes/bin/log-counter
-
+ADD ./bin/log-counter-BINARCH /home/kubernetes/bin/log-counter
 ADD config /config
 ENTRYPOINT ["/node-problem-detector", "--system-log-monitors=/config/kernel-monitor.json"]

Makefile

@@ -15,8 +15,13 @@
 # Build the node-problem-detector image.
 
 .PHONY: all build-container build-tar build push-container push-tar push \
-        clean vet fmt version \
-        Dockerfile build-binaries docker-builder build-in-docker
+		clean vet fmt version \
+        Dockerfile build-binaries docker-builder build-in-docker \
+		build-all build-all-container \
+        build-tar-all push-all push-all-container push-manifest push-tar-all qemu-register \
+		release-staging
+
+TEMP_DIR := $(shell mktemp -d)
 
 all: build
 
@@ -26,9 +31,19 @@ VERSION?=$(shell if [ -d .git ]; then echo `git describe --tags --dirty`; else e
 # TAG is the tag of the container image, default to binary version.
 TAG?=$(VERSION)
 
+ARCH ?= $(shell go env GOARCH)
+ALL_ARCH = amd64 arm arm64 ppc64le s390x
+QEMUVERSION=v4.0.0-4
+
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
+
 # REGISTRY is the container registry to push into.
 REGISTRY?=staging-k8s.gcr.io
 
+# STAGING_REGISTRY is the new registry for staging
+STAGING_REGISTRY?=gcr.io/k8s-staging-npd
+
 # UPLOAD_PATH is the cloud storage path to upload release tar.
 UPLOAD_PATH?=gs://kubernetes-release
 # Trim the trailing '/' in the path
@@ -44,20 +59,45 @@ PKG_SOURCES:=$(shell find pkg cmd -name '*.go')
 PARALLEL?=3
 
 # TARBALL is the name of release tar. Include binary version by default.
-TARBALL?=node-problem-detector-$(VERSION).tar.gz
+TARBALL:=node-problem-detector-$(VERSION).tar.gz
+TARBALL-ALL:=node-problem-detector-$(VERSION)-all.tar.gz
 
 # IMAGE is the image name of the node problem detector container image.
 IMAGE:=$(REGISTRY)/node-problem-detector:$(TAG)
+IMAGE_WITH_ARCH:=$(IMAGE)-$(ARCH)
 
 # ENABLE_JOURNALD enables build journald support or not. Building journald
 # support needs libsystemd-dev or libsystemd-journal-dev.
 ENABLE_JOURNALD?=1
 
-# TODO(random-liu): Support different architectures.
 # The debian-base:v1.0.0 image built from kubernetes repository is based on
 # Debian Stretch. It includes systemd 232 with support for both +XZ and +LZ4
 # compression. +LZ4 is needed on some os distros such as COS.
-BASEIMAGE:=k8s.gcr.io/debian-base-amd64:v1.0.0
+# Set the (cross) compiler, BASEIMAGE to use for different architectures
+ifeq ($(ARCH),amd64)
+	CC=gcc
+	BASEIMAGE:=k8s.gcr.io/debian-base-amd64:v1.0.0
+endif
+ifeq ($(ARCH),arm)
+	QEMUARCH=arm
+	CC=arm-linux-gnueabihf-gcc
+	BASEIMAGE:=k8s.gcr.io/debian-base-arm:v1.0.0
+endif
+ifeq ($(ARCH),arm64)
+	QEMUARCH=aarch64
+	CC=aarch64-linux-gnu-gcc
+	BASEIMAGE:=k8s.gcr.io/debian-base-arm64:v1.0.0
+endif
+ifeq ($(ARCH),ppc64le)
+	QEMUARCH=ppc64le
+	CC=powerpc64le-linux-gnu-gcc
+	BASEIMAGE:=k8s.gcr.io/debian-base-ppc64le:v1.0.0
+endif
+ifeq ($(ARCH),s390x)
+	QEMUARCH=s390x
+	CC=s390x-linux-gnu-gcc
+	BASEIMAGE:=k8s.gcr.io/debian-base-s390x:v1.0.0
+endif
 
 # Disable cgo by default to make the binary statically linked.
 CGO_ENABLED:=0
@@ -88,9 +128,9 @@ version:
 
 ./bin/log-counter: $(PKG_SOURCES)
 ifeq ($(ENABLE_JOURNALD), 1)
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GO111MODULE=on go build \
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=$(ARCH) GO111MODULE=on go build \
 		-mod vendor \
-		-o bin/log-counter \
+		-o bin/log-counter-$(ARCH) \
 		-ldflags '-X $(PKG)/pkg/version.version=$(VERSION)' \
 		-tags "$(BUILD_TAGS)" \
 		cmd/logcounter/log_counter.go
@@ -99,27 +139,38 @@ else
 endif
 
 ./bin/node-problem-detector: $(PKG_SOURCES)
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GO111MODULE=on go build \
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=$(ARCH) GO111MODULE=on go build \
 		-mod vendor \
-		-o bin/node-problem-detector \
+		-o bin/node-problem-detector-$(ARCH) \
 		-ldflags '-X $(PKG)/pkg/version.version=$(VERSION)' \
 		-tags "$(BUILD_TAGS)" \
 		./cmd/nodeproblemdetector
 
 ./test/bin/problem-maker: $(PKG_SOURCES)
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GO111MODULE=on go build \
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=$(ARCH) GO111MODULE=on go build \
 		-mod vendor \
 		-o test/bin/problem-maker \
 		-tags "$(BUILD_TAGS)" \
 		./test/e2e/problemmaker/problem_maker.go
 
+qemu-register:
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
 Dockerfile: Dockerfile.in
 	sed -e 's|@BASEIMAGE@|$(BASEIMAGE)|g' $< >$@
 ifneq ($(ENABLE_JOURNALD), 1)
 	sed -i '/Below command depends on ENABLE_JOURNAL=1/,+2d' $@
 	echo "Warning: log-counter requires journald, skipping."
 endif
-
+	sed -i "s|-ARCH|-$(QEMUARCH)|g" $@
+	sed -i "s|-BINARCH|-$(ARCH)|g" $@
+ifeq ($(ARCH),amd64)
+	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	sed -i "/CROSS_BUILD_/d" Dockerfile
+else
+	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C .
+	sed -i "s/CROSS_BUILD_//g" Dockerfile
+endif
 
 test: vet fmt
 	GO111MODULE=on go test -mod vendor -timeout=1m -v -race -short -tags "$(BUILD_TAGS)" ./...
@@ -136,35 +187,71 @@ e2e-test: vet fmt build-tar
 
 build-binaries: ./bin/node-problem-detector ./bin/log-counter
 
-build-container: build-binaries Dockerfile
-	docker build -t $(IMAGE) .
+build-all-container: $(addprefix build-container-,$(ALL_ARCH))
+
+build-container-%: qemu-register
+	$(MAKE) ARCH=$* build-container
+
+build-container: clean build-in-docker Dockerfile
+	docker build -t $(IMAGE_WITH_ARCH) .
 
 build-tar: ./bin/node-problem-detector ./bin/log-counter ./test/bin/problem-maker
 	tar -zcvf $(TARBALL) bin/ config/ test/e2e-install.sh test/bin/problem-maker
 	sha1sum $(TARBALL)
 	md5sum $(TARBALL)
 
+build-tar-all: clean
+	@for arch in $(ALL_ARCH); do $(MAKE) ARCH=$${arch} build-in-docker; done
+	@for arch in $(ALL_ARCH); do cp bin/node-problem-detector-$(ARCH) bin/node-problem-detector; done
+	@for arch in $(ALL_ARCH); do cp bin/log-counter-$(ARCH) bin/log-counter; done
+	tar -zcvf $(TARBALL-ALL) bin/ config/ test/e2e-install.sh
+	sha1sum $(TARBALL-ALL)
+	md5sum $(TARBALL-ALL)
+
+build-all: build-all-container build-tar-all
+
 build: build-container build-tar
 
 docker-builder:
 	docker build -t npd-builder ./builder
 
-build-in-docker: clean docker-builder
+build-in-docker: docker-builder
 	docker run \
+		-e CC=$(CC) -e GOARM=$(GOARM) -e GOARCH=$(ARCH) -e GOCACHE=/go/.cache/go-build \
+		-u $(shell id -u ${USER}):$(shell id -g ${USER}) \
 		-v `pwd`:/gopath/src/k8s.io/node-problem-detector/ npd-builder:latest bash \
 		-c 'cd /gopath/src/k8s.io/node-problem-detector/ && make build-binaries'
 
+push-all-container: build-all-container $(addprefix push-container-,$(ALL_ARCH))
+
+push-container-%:
+	$(MAKE) ARCH=$* push-container
+
 push-container: build-container
 	gcloud auth configure-docker
-	docker push $(IMAGE)
+	docker push $(IMAGE_WITH_ARCH)
+
+push-manifest:
+	gcloud auth configure-docker
+	docker manifest create --amend $(IMAGE) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(IMAGE)\-&~g")
+	@for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${IMAGE} ${IMAGE}-$${arch}; done
+	docker manifest push --purge ${IMAGE}
 
 push-tar: build-tar
 	gsutil cp $(TARBALL) $(UPLOAD_PATH)/node-problem-detector/
 
+push-tar-all: build-tar-all
+	gsutil cp $(TARBALL-ALL) $(UPLOAD_PATH)/node-problem-detector/
+
+push-all: push-all-container push-manifest push-tar-all push-tar
+
 push: push-container push-tar
 
+release-staging:
+	REGISTRY=$(STAGING_REGISTRY) $(MAKE) push-all
+
 clean:
-	rm -f bin/log-counter
-	rm -f bin/node-problem-detector
-	rm -f test/bin/problem-maker
+	rm -f bin/log-counter*
+	rm -f bin/node-problem-detector*
+	rm -f test/bin/problem-maker*
 	rm -f node-problem-detector-*.tar.gz

builder/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.11.0
+FROM gcr.io/google_containers/kube-cross:v1.12.9-1
 LABEL maintainer="Andy Xie <[email protected]>"
 
 ENV GOPATH /gopath/

cloudbuild.yaml

@@ -0,0 +1,17 @@
+timeout: 1200s
+options:
+  substitution_option: ALLOW_LOOSE
+steps:
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4'
+    entrypoint: make
+    env:
+    - TAG=$_GIT_TAG
+    - BASE_REF=$_PULL_BASE_REF
+    - DOCKER_CLI_EXPERIMENTAL=enabled
+    args:
+    - release-staging
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution
+  _GIT_TAG: '12345'
+  _PULL_BASE_REF: 'dev'

test/build.sh

@@ -131,7 +131,7 @@ function build-pr() {
   export REGISTRY="${NPD_STAGING_REGISTRY}/pr/${PR}"
   export VERSION=$(get-version)
   export TAG="${VERSION}"
-  make push
+  make push-all
   write-env-file ${PR_ENV_FILENAME}
 }
 
@@ -141,7 +141,7 @@ function build-ci() {
   export REGISTRY="${NPD_STAGING_REGISTRY}/ci"
   export VERSION="$(get-version)-$(date +%Y%m%d.%H%M)"
   export TAG="${VERSION}"
-  make push
+  make push-all
 
   # Create the env file with and without custom flags at the same time.
   build-npd-custom-flags

test/e2e-install.sh

@@ -55,10 +55,10 @@ function install-npd() {
   mount -o remount,exec "${BIN_DIR}"
 
   echo "Installing NPD binary."
-  cp "${workdir}"/bin/node-problem-detector "${BIN_DIR}"
+  cp "${workdir}"/bin/node-problem-detector-amd64 "${BIN_DIR}"/node-problem-detector
 
   echo "Installing log-counter binary."
-  cp "${workdir}"/bin/log-counter "${BIN_DIR}"
+  cp "${workdir}"/bin/log-counter-amd64 "${BIN_DIR}"/log-counter
 
   echo "Installing NPD configurations."
   mkdir -p "${CONFIG_DIR}"
@@ -97,4 +97,4 @@ done
 shift "$((OPTIND-1))"
 
 
-main "${@}"
+main "${@}"