From 0c2809717cf0f104062e4bcc7e0e08f16828610b Mon Sep 17 00:00:00 2001
From: Blaine Gardner <blaine.gardner@ibm.com>
Date: Mon, 9 Sep 2024 18:30:48 -0600
Subject: [PATCH] set up new release tooling for controller/sidecar subrepos

Set up new cloudbuild tooling for the controller/sidecar subrepo layout.

Controller and sidecar are both always built pushed to staging  with the
current git tag (e.g., v20240905-v0.1.0-58-g80979e8) as well as 'latest'.

Controller pre-release staging build can be initiated by creating a
github release (or tag) in the form `cloudbuild/TAG`.

Similarly, sidecar pre-release staging build can be initiated by creating
a github release (or tag) in the form `sidecar/TAG`.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
---
 .prow.sh                   |  6 -----
 Makefile                   | 11 ++++++---
 cloudbuild.yaml            | 27 ++++++++++++++++++++++
 controller/cloudbuild.yaml | 39 --------------------------------
 hack/cloudbuild.sh         | 46 ++++++++++++++++++++++++++++++++++++++
 sidecar/cloudbuild.yaml    | 39 --------------------------------
 6 files changed, 81 insertions(+), 87 deletions(-)
 delete mode 100755 .prow.sh
 create mode 100644 cloudbuild.yaml
 delete mode 100644 controller/cloudbuild.yaml
 create mode 100644 hack/cloudbuild.sh
 delete mode 100644 sidecar/cloudbuild.yaml

diff --git a/.prow.sh b/.prow.sh
deleted file mode 100755
index 682601dd..00000000
--- a/.prow.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#! /bin/bash
-
-. release-tools/prow.sh
-
-main
-
diff --git a/Makefile b/Makefile
index 8e78ef42..7c214485 100644
--- a/Makefile
+++ b/Makefile
@@ -20,6 +20,9 @@ SHELL = /usr/bin/env bash
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+# If GOARCH is not set in the env, find it
+GOARCH ?= $(shell go env GOARCH)
+
 ##
 ## ==== ARGS ===== #
 
@@ -29,13 +32,15 @@ DOCKER ?= docker
 ## Platform for 'build'
 PLATFORM ?= linux/$(GOARCH)
 
+## Additional args for 'build'
+BUILD_ARGS ?=
+
 ## Image tag for controller image build
 CONTROLLER_TAG ?= cosi-controller:latest
 
 ## Image tag for sidecar image build
 SIDECAR_TAG ?= cosi-provisioner-sidecar:latest
 
-
 ##@ Development
 
 .PHONY: all .gen
@@ -68,9 +73,9 @@ build: build.controller build.sidecar ## Build all container images for developm
 
 .PHONY: build.controller build.sidecar
 build.controller: controller/Dockerfile ## Build only the controller container image
-	$(DOCKER) build --file controller/Dockerfile --platform $(PLATFORM) --tag $(CONTROLLER_TAG) .
+	$(DOCKER) build --file controller/Dockerfile --platform $(PLATFORM) $(BUILD_ARGS) --tag $(CONTROLLER_TAG) .
 build.sidecar: sidecar/Dockerfile ## Build only the sidecar container image
-	$(DOCKER) build --file sidecar/Dockerfile --platform $(PLATFORM) --tag $(SIDECAR_TAG) .
+	$(DOCKER) build --file sidecar/Dockerfile --platform $(PLATFORM) $(BUILD_ARGS) --tag $(SIDECAR_TAG) .
 
 .PHONY: clean
 ## Clean build environment
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
new file mode 100644
index 00000000..3e91d695
--- /dev/null
+++ b/cloudbuild.yaml
@@ -0,0 +1,27 @@
+# K8s infra build example: https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
+# GCloud build docs: https://cloud.google.com/cloud-build/docs/build-config
+# Builds go to https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL
+# Build logs in https://testgrid.k8s.io/sig-storage-image-build
+timeout: 3600s
+options:
+  substitution_option: 'ALLOW_LOOSE'
+  machineType: 'E2_HIGHCPU_8'
+substitutions:
+  # K8s provides custom substitutions _GIT_TAG and _PULL_BASE_REF:
+  #   https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md#custom-substitutions
+  _GIT_TAG: '12345'  # e.g., vYYYYMMDD-hash, vYYYYMMDD-tag, or vYYYYMMDD-tag-n-ghash
+  _PULL_BASE_REF: 'master'  # e.g., master or release-0.2 for a PR merge, or v0.2 for a tag
+  # COSI substitutions:
+  _PLATFORMS: linux/amd64,linux/arm64 # add more platforms here if desired
+steps:
+  # TODO: currently gcr.io/k8s-testimages/gcb-docker-gcloud has not moved to Artifact Registry
+  # gcr.io will be shut down 18 Mar 2025, and we need replacement before then. Latest info below:
+  # https://github.com/kubernetes/test-infra/blob/master/images/gcb-docker-gcloud/cloudbuild.yaml
+  - id: do-multi-arch-build-all-images
+    name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20240718-5ef92b5c36
+    args:
+      - hack/cloudbuild.sh
+    env:
+      - GIT_TAG=$_GIT_TAG
+      - PULL_BASE_REF=$_PULL_BASE_REF
+      - PLATFORM=$_PLATFORMS
diff --git a/controller/cloudbuild.yaml b/controller/cloudbuild.yaml
deleted file mode 100644
index bd1d6886..00000000
--- a/controller/cloudbuild.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-# K8s infra build example: https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
-# GCloud build docs: https://cloud.google.com/cloud-build/docs/build-config
-# Builds go to https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL
-# Build logs in https://testgrid.k8s.io/sig-storage-image-build
-timeout: 3000s
-options:
-  substitution_option: 'ALLOW_LOOSE'
-  machineType: 'E2_HIGHCPU_8'
-substitutions:
-  # K8s provides custom substitutions _GIT_TAG and _PULL_BASE_REF:
-  #   https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md#custom-substitutions
-  _GIT_TAG: '12345'  # e.g., vYYYYMMDD-hash, vYYYYMMDD-tag, or vYYYYMMDD-tag-n-ghash
-  _PULL_BASE_REF: 'master'  # e.g., master or release-0.2 for a PR merge, or v0.2 for a tag
-  # COSI substitutions:
-  _IMAGE: objectstorage-controller
-  _PLATFORMS: linux/amd64,linux/arm64 # add more platforms here if desired
-steps:
-  # based on k8s infra build example and modified to use docker buildx
-  # buildx refs: https://github.com/docker/buildx/blob/master/docs/reference/buildx.md
-  - id: set-up-multi-arch-builder-as-default
-    name: gcr.io/cloud-builders/docker
-    args: ["buildx", "create", "--use", "--name", "multi-arch", "--platform=${_PLATFORMS}"]
-  - id: bootstrap-multi-arch-builder
-    name: gcr.io/cloud-builders/docker
-    args: ["buildx", "inspect", "--bootstrap"]
-  - id: do-multi-arch-build
-    name: gcr.io/cloud-builders/docker
-    args:
-      - buildx
-      - build
-      - --push # --load doesn't work with multi-platform builds, so just push from docker build
-      - --platform=${_PLATFORMS}
-      - --tag=gcr.io/k8s-staging-sig-storage/${_IMAGE}:${_GIT_TAG}
-      # using _PULL_BASE_REF as a tag will often just build and overwrite the same 'master' tag,
-      # BUT! if the commit has a git tag, it will build that tag instead. this mechanism allows
-      # creating the semver-tagged images that will be auto-promoted to release
-      - --tag=gcr.io/k8s-staging-sig-storage/${_IMAGE}:${_PULL_BASE_REF}
-      - --tag=gcr.io/k8s-staging-sig-storage/${_IMAGE}:latest
-      - .
diff --git a/hack/cloudbuild.sh b/hack/cloudbuild.sh
new file mode 100644
index 00000000..1dffdae8
--- /dev/null
+++ b/hack/cloudbuild.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+
+# with nounset, these will fail if necessary vars are missing
+echo "GIT_TAG: ${GIT_TAG}"
+echo "PULL_BASE_REF: ${PULL_BASE_REF}"
+echo "PLATFORM: ${PLATFORM}"
+
+# debug the rest of the script in case of image/CI build issues
+set -o xtrace
+
+REPO="gcr.io/k8s-staging-sig-storage"
+
+CONTROLLER_IMAGE="${REPO}/objectstorage-controller"
+SIDECAR_IMAGE="${REPO}/objectstorage-sidecar"
+
+# args to 'make build'
+export DOCKER="/buildx-entrypoint" # available in gcr.io/k8s-testimages/gcb-docker-gcloud image
+export BUILD_ARGS="--push"
+export PLATFORM
+export SIDECAR_TAG="${SIDECAR_IMAGE}:${GIT_TAG}"
+export CONTROLLER_TAG="${CONTROLLER_IMAGE}:${GIT_TAG}"
+
+# build in parallel
+make --jobs --output-sync build
+
+# add latest tag to just-built images
+gcloud container images add-tag "${CONTROLLER_TAG}" "${CONTROLLER_IMAGE}:latest"
+gcloud container images add-tag "${SIDECAR_TAG}" "${SIDECAR_IMAGE}:latest"
+
+# PULL_BASE_REF is 'controller/TAG' for a controller release
+if [[ "${PULL_BASE_REF}" == controller/* ]]; then
+  echo " ! ! ! this is a tagged controller release ! ! !"
+  TAG="${PULL_BASE_REF#controller/*}"
+  gcloud container images add-tag "${CONTROLLER_TAG}" "${CONTROLLER_IMAGE}:${TAG}"
+fi
+
+# PULL_BASE_REF is 'sidecar/TAG' for a controller release
+if [[ "${PULL_BASE_REF}" == sidecar/* ]]; then
+  echo " ! ! ! this is a tagged sidecar release ! ! !"
+  TAG="${PULL_BASE_REF#sidecar/*}"
+  gcloud container images add-tag "${SIDECAR_TAG}" "${SIDECAR_IMAGE}:${TAG}"
+fi
+
+# else, PULL_BASE_REF is a branch name (e.g., master, release-0.2) or a tag (e.g., client/v0.2.0, proto/v0.2.0)
diff --git a/sidecar/cloudbuild.yaml b/sidecar/cloudbuild.yaml
deleted file mode 100644
index d2cc261d..00000000
--- a/sidecar/cloudbuild.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-# K8s infra build example: https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
-# GCloud build docs: https://cloud.google.com/cloud-build/docs/build-config
-# Builds go to https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL
-# Build logs in https://testgrid.k8s.io/sig-storage-image-build
-timeout: 3000s
-options:
-  substitution_option: 'ALLOW_LOOSE'
-  machineType: 'E2_HIGHCPU_8'
-substitutions:
-  # K8s provides custom substitutions _GIT_TAG and _PULL_BASE_REF:
-  #   https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md#custom-substitutions
-  _GIT_TAG: '12345'  # e.g., vYYYYMMDD-hash, vYYYYMMDD-tag, or vYYYYMMDD-tag-n-ghash
-  _PULL_BASE_REF: 'master'  # e.g., master or release-0.2 for a PR merge, or v0.2 for a tag
-  # COSI substitutions:
-  _IMAGE: objectstorage-sidecar
-  _PLATFORMS: linux/amd64,linux/arm64 # add more platforms here if desired
-steps:
-  # based on k8s infra build example and modified to use docker buildx
-  # buildx refs: https://github.com/docker/buildx/blob/master/docs/reference/buildx.md
-  - id: set-up-multi-arch-builder-as-default
-    name: gcr.io/cloud-builders/docker
-    args: ["buildx", "create", "--use", "--name", "multi-arch", "--platform=${_PLATFORMS}"]
-  - id: bootstrap-multi-arch-builder
-    name: gcr.io/cloud-builders/docker
-    args: ["buildx", "inspect", "--bootstrap"]
-  - id: do-multi-arch-build
-    name: gcr.io/cloud-builders/docker
-    args:
-      - buildx
-      - build
-      - --push # --load doesn't work with multi-platform builds, so just push from docker build
-      - --platform=${_PLATFORMS}
-      - --tag=gcr.io/k8s-staging-sig-storage/${_IMAGE}:${_GIT_TAG}
-      # using _PULL_BASE_REF as a tag will often just build and overwrite the same 'master' tag,
-      # BUT! if the commit has a git tag, it will build that tag instead. this mechanism allows
-      # creating the semver-tagged images that will be auto-promoted to release
-      - --tag=gcr.io/k8s-staging-sig-storage/${_IMAGE}:${_PULL_BASE_REF}
-      - --tag=gcr.io/k8s-staging-sig-storage/${_IMAGE}:latest
-      - .