From c9e322aa32991c65e11919b8e4ea6ec5b7ecea36 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 9 Jan 2023 16:58:49 -0800 Subject: [PATCH 1/8] chore: Use Github environment for CI secrets --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43112040b..34413c350 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ permissions: jobs: staticAnalysis: name: Static Analysis + environment: ci runs-on: ubuntu-latest steps: - name: Configure AWS Credentials @@ -30,6 +31,7 @@ jobs: image-override: aws/codebuild/amazonlinux2-x86_64-standard:3.0 vectorTests: name: Vector Tests + environment: ci runs-on: ubuntu-latest strategy: fail-fast: true @@ -60,6 +62,7 @@ jobs: JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }} releaseCI: name: Release CI + environment: ci runs-on: ubuntu-latest steps: - name: Configure AWS Credentials @@ -78,6 +81,7 @@ jobs: image-override: aws/codebuild/standard:3.0 validateCI: name: Validate CI + environment: ci runs-on: ubuntu-latest needs: releaseCI strategy: From a8039cc31891785be2233f2d2ae7b9d8eebc193a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 12 Jan 2023 16:19:32 -0800 Subject: [PATCH 2/8] chore: Use AWS CLI to start CodeBuild batch --- .github/scripts/poll_build_status.sh | 31 +++++++ .github/scripts/start_codebuild_batch.sh | 9 ++ .github/workflows/ci.yml | 106 +++-------------------- 3 files changed, 51 insertions(+), 95 deletions(-) create mode 100644 .github/scripts/poll_build_status.sh create mode 100644 .github/scripts/start_codebuild_batch.sh diff --git a/.github/scripts/poll_build_status.sh b/.github/scripts/poll_build_status.sh new file mode 100644 index 000000000..d07e72c89 --- /dev/null +++ b/.github/scripts/poll_build_status.sh @@ -0,0 +1,31 @@ +# Polls the provided CodeBuild batch ID until the build terminates +# Script terminates when the build status is not "IN_PROGRESS". +# +# Usage: ./poll_build_status.sh [batch-build-id] + +BUILD_TIMEOUT_MINUTES=120 +POLL_COUNTER=0 + +while [ $POLL_COUNTER -lt $BUILD_TIMEOUT_MINUTES ]; do + BUILD_STATUS=$(aws codebuild batch-get-build-batches \ + --ids "$1" \ + | jq '.buildBatches.buildBatchStatus') + + echo "Build status is $BUILD_STATUS after $POLL_COUNTER minutes" + # If build succeeds, exit 0; Github will interpret 'exit 0' as successful job run + if [ "$BUILD_STATUS" == "SUCCEEDED" ]; then + exit 0 + fi + + # If build is not successful nor in-progress, it has either failed, timed-out, faulted, or been stopped. + # Github will interpret 'exit 1' as job failure + if [ "$BUILD_STATUS" != "IN_PROGRESS" ]; then + exit 1 + fi + + ((POLL_COUNTER++)) + sleep 60 +done + +# If job does not report success within BUILD_TIMEOUT_MINUTES, fail Github job +exit 1; \ No newline at end of file diff --git a/.github/scripts/start_codebuild_batch.sh b/.github/scripts/start_codebuild_batch.sh new file mode 100644 index 000000000..4df69d9be --- /dev/null +++ b/.github/scripts/start_codebuild_batch.sh @@ -0,0 +1,9 @@ +# Starts a CodeBuild batch with provided source version +# Returns build batch ID for build +# +# Usage: ./start_codebuild_batch.sh [source_version] +aws codebuild start-build-batch \ + --region us-west-2 \ + --project-name AWS-ESDK-Java-CI \ + --source-version "$1" \ + | jq '.buildBatch.id' \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34413c350..417c1b6fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,8 @@ permissions: contents: read jobs: - staticAnalysis: - name: Static Analysis + runCodeBuildCI: + name: Run CodeBuild CI environment: ci runs-on: ubuntu-latest steps: @@ -21,96 +21,12 @@ jobs: role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} aws-region: us-west-2 role-duration-seconds: 3600 - - name: Run Static Analysis - uses: aws-actions/aws-codebuild-run-build@v1 - timeout-minutes: 60 - with: - project-name: AWS-ESDK-Java-CI - buildspec-override: codebuild/ci/static-analysis.yml - compute-type-override: BUILD_GENERAL1_MEDIUM - image-override: aws/codebuild/amazonlinux2-x86_64-standard:3.0 - vectorTests: - name: Vector Tests - environment: ci - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - platform: - - distribution: openjdk - image: "aws/codebuild/standard:3.0" - - distribution: corretto - image: "aws/codebuild/amazonlinux2-x86_64-standard:3.0" # Corretto only runs on AL2 - version: [ 8, 11 ] - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 3600 - - name: Vector Tests ${{ matrix.platform.distribution }}${{ matrix.version }} - uses: aws-actions/aws-codebuild-run-build@v1 - timeout-minutes: 60 - with: - project-name: AWS-ESDK-Java-CI - buildspec-override: codebuild/ci/vectors-ci.yml - compute-type-override: BUILD_GENERAL1_LARGE - image-override: ${{ matrix.platform.image }} - env-vars-for-codebuild: JAVA_ENV_VERSION - env: - JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }} - releaseCI: - name: Release CI - environment: ci - runs-on: ubuntu-latest - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 3600 - - name: Release CI - uses: aws-actions/aws-codebuild-run-build@v1 - timeout-minutes: 60 - with: - project-name: AWS-ESDK-Java-CI - buildspec-override: codebuild/ci/release-ci.yml - compute-type-override: BUILD_GENERAL1_LARGE - image-override: aws/codebuild/standard:3.0 - validateCI: - name: Validate CI - environment: ci - runs-on: ubuntu-latest - needs: releaseCI - strategy: - fail-fast: true - matrix: - platform: - - distribution: openjdk - image: "aws/codebuild/standard:3.0" - - distribution: corretto - image: "aws/codebuild/amazonlinux2-x86_64-standard:3.0" # Corretto only runs on AL2 - version: [ 8, 11 ] - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} - aws-region: us-west-2 - role-duration-seconds: 3600 - - name: Validate CI ${{ matrix.platform.distribution }}${{ matrix.version }} - uses: aws-actions/aws-codebuild-run-build@v1 - timeout-minutes: 60 - with: - project-name: AWS-ESDK-Java-CI - buildspec-override: codebuild/ci/validate-ci.yml - compute-type-override: BUILD_GENERAL1_MEDIUM - image-override: ${{ matrix.platform.image }} - env-vars-for-codebuild: | - JAVA_ENV_VERSION, - JAVA_NUMERIC_VERSION - env: - JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }} - JAVA_NUMERIC_VERSION: ${{ matrix.version }} + - name: Start CodeBuild Batch Build + run: | + echo 'BATCH_ID<> $GITHUB_ENV + ./.github/scripts/start_codebuild_batch.sh pr/${{ github.event.number }} >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + shell: bash + - name: Wait for CodeBuild completion + run: ./.github/scripts/poll_build_status.sh ${{ env.BATCH_ID }} + shell: bash \ No newline at end of file From 439358757311453f8c6238cac10dc778593e61e0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 12 Jan 2023 16:25:26 -0800 Subject: [PATCH 3/8] chore: Use commit ID instead of PR number --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 417c1b6fb..42bf854c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Start CodeBuild Batch Build run: | echo 'BATCH_ID<> $GITHUB_ENV - ./.github/scripts/start_codebuild_batch.sh pr/${{ github.event.number }} >> $GITHUB_ENV + ./.github/scripts/start_codebuild_batch.sh ${{ env.GITHUB_SHA }} >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV shell: bash - name: Wait for CodeBuild completion From a1fb888fdd88d18678320b19ded19cb8d99ad3ee Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 13 Jan 2023 10:57:51 -0800 Subject: [PATCH 4/8] chore: Update CI script --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42bf854c5..52776b8fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,11 +21,13 @@ jobs: role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} aws-region: us-west-2 role-duration-seconds: 3600 + - name: Checkout + uses: actions/checkout@v3 - name: Start CodeBuild Batch Build run: | - echo 'BATCH_ID<> $GITHUB_ENV - ./.github/scripts/start_codebuild_batch.sh ${{ env.GITHUB_SHA }} >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV + echo "BATCH_ID<> $GITHUB_ENV + ./.github/scripts/start_codebuild_batch.sh ${{ GITHUB_SHA }} >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV shell: bash - name: Wait for CodeBuild completion run: ./.github/scripts/poll_build_status.sh ${{ env.BATCH_ID }} From 2b275559c655e5fdbe404d464dc65d7eef70806a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 13 Jan 2023 10:59:03 -0800 Subject: [PATCH 5/8] chore: Update SHA access --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52776b8fb..d0de7c903 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Start CodeBuild Batch Build run: | echo "BATCH_ID<> $GITHUB_ENV - ./.github/scripts/start_codebuild_batch.sh ${{ GITHUB_SHA }} >> $GITHUB_ENV + ./.github/scripts/start_codebuild_batch.sh ${{ github.sha }} >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV shell: bash - name: Wait for CodeBuild completion From f50746c183805947400adc461ad2693e8a11494d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 13 Jan 2023 11:22:44 -0800 Subject: [PATCH 6/8] chore: Update script permissions --- .github/scripts/poll_build_status.sh | 0 .github/scripts/start_codebuild_batch.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/scripts/poll_build_status.sh mode change 100644 => 100755 .github/scripts/start_codebuild_batch.sh diff --git a/.github/scripts/poll_build_status.sh b/.github/scripts/poll_build_status.sh old mode 100644 new mode 100755 diff --git a/.github/scripts/start_codebuild_batch.sh b/.github/scripts/start_codebuild_batch.sh old mode 100644 new mode 100755 From 80ee5cfb670ec87cec0010fcb5dea9a8f174c498 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 18 Jan 2023 10:41:40 -0800 Subject: [PATCH 7/8] chore: Fix CLI scripts --- .github/scripts/poll_build_status.sh | 4 +- .github/scripts/start_codebuild_batch.sh | 2 +- codebuild/ci/ci.yml | 89 ++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 codebuild/ci/ci.yml diff --git a/.github/scripts/poll_build_status.sh b/.github/scripts/poll_build_status.sh index d07e72c89..3068fed8c 100755 --- a/.github/scripts/poll_build_status.sh +++ b/.github/scripts/poll_build_status.sh @@ -9,7 +9,9 @@ POLL_COUNTER=0 while [ $POLL_COUNTER -lt $BUILD_TIMEOUT_MINUTES ]; do BUILD_STATUS=$(aws codebuild batch-get-build-batches \ --ids "$1" \ - | jq '.buildBatches.buildBatchStatus') + | jq -r --arg BATCH_ID "$1" '.buildBatches[] + | select(.id == $BATCH_ID) + | .buildBatchStatus' echo "Build status is $BUILD_STATUS after $POLL_COUNTER minutes" # If build succeeds, exit 0; Github will interpret 'exit 0' as successful job run diff --git a/.github/scripts/start_codebuild_batch.sh b/.github/scripts/start_codebuild_batch.sh index 4df69d9be..9700f822b 100755 --- a/.github/scripts/start_codebuild_batch.sh +++ b/.github/scripts/start_codebuild_batch.sh @@ -6,4 +6,4 @@ aws codebuild start-build-batch \ --region us-west-2 \ --project-name AWS-ESDK-Java-CI \ --source-version "$1" \ - | jq '.buildBatch.id' \ No newline at end of file + | jq -r '.buildBatch.id' \ No newline at end of file diff --git a/codebuild/ci/ci.yml b/codebuild/ci/ci.yml new file mode 100644 index 000000000..3b74de5a9 --- /dev/null +++ b/codebuild/ci/ci.yml @@ -0,0 +1,89 @@ +version: 0.2 + +# TODO: Replace/Augment build-graph with build-matrix +# Note: It's possible that 9 builds will be running concurrently with the current arrangement + +batch: + fast-fail: false + build-graph: + - identifier: static_analysis + buildspec: codebuild/ci/static-analysis.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 + ######### Start Vector tests ######### + - identifier: vectors_ci_openjdk8 + buildspec: codebuild/ci/vectors-ci.yml + env: + compute-type: BUILD_GENERAL1_LARGE + variables: + JAVA_ENV_VERSION: openjdk8 + image: aws/codebuild/standard:3.0 + - identifier: vectors_ci_openjdk11 + buildspec: codebuild/ci/vectors-ci.yml + env: + compute-type: BUILD_GENERAL1_LARGE + variables: + JAVA_ENV_VERSION: openjdk11 + image: aws/codebuild/standard:3.0 + - identifier: vectors_ci_corretto8 + buildspec: codebuild/ci/vectors-ci.yml + env: + compute-type: BUILD_GENERAL1_LARGE + variables: + JAVA_ENV_VERSION: corretto8 + image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 + - identifier: vectors_ci_corretto11 + buildspec: codebuild/ci/vectors-ci.yml + env: + compute-type: BUILD_GENERAL1_LARGE + variables: + JAVA_ENV_VERSION: corretto11 + image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 + ######### End Vector tests ######### + - identifier: release_ci + buildspec: codebuild/ci/release-ci.yml + env: + compute-type: BUILD_GENERAL1_LARGE + image: aws/codebuild/standard:3.0 + ######### Start JAR Smoke tests ######### + - identifier: validate_ci_openjdk8 + depend-on: + - release_ci + buildspec: codebuild/ci/validate-ci.yml + env: + variables: + JAVA_ENV_VERSION: openjdk8 + JAVA_NUMERIC_VERSION: 8 + image: aws/codebuild/standard:3.0 + - identifier: validate_ci_openjdk11 + depend-on: + - release_ci + buildspec: codebuild/ci/validate-ci.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + JAVA_ENV_VERSION: openjdk11 + JAVA_NUMERIC_VERSION: 11 + image: aws/codebuild/standard:3.0 + - identifier: validate_ci_corretto8 + depend-on: + - release_ci + buildspec: codebuild/ci/validate-ci.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + JAVA_ENV_VERSION: corretto8 + JAVA_NUMERIC_VERSION: 8 + image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 + - identifier: validate_ci_corretto11 + depend-on: + - release_ci + buildspec: codebuild/ci/validate-ci.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + JAVA_ENV_VERSION: corretto11 + JAVA_NUMERIC_VERSION: 11 + image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 + ######### End JAR Smoke tests ######### \ No newline at end of file From f17d31ff36159d6aebc79537b7bf9cc02e0771be Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 18 Jan 2023 11:29:32 -0800 Subject: [PATCH 8/8] chore: Fix build script syntax --- .github/scripts/poll_build_status.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/poll_build_status.sh b/.github/scripts/poll_build_status.sh index 3068fed8c..e1c097681 100755 --- a/.github/scripts/poll_build_status.sh +++ b/.github/scripts/poll_build_status.sh @@ -11,7 +11,7 @@ while [ $POLL_COUNTER -lt $BUILD_TIMEOUT_MINUTES ]; do --ids "$1" \ | jq -r --arg BATCH_ID "$1" '.buildBatches[] | select(.id == $BATCH_ID) - | .buildBatchStatus' + | .buildBatchStatus' ) echo "Build status is $BUILD_STATUS after $POLL_COUNTER minutes" # If build succeeds, exit 0; Github will interpret 'exit 0' as successful job run