Skip to content

Commit 7ae9569

Browse files
authored
Prevent Windows GHA CI disk exhausation (#468)
Windows BoringSSL builds are now larger due to #454. The checkout & build partition **(D:)** hits capacity when attempting to install the Google Cloud SDK, which is what we use to upload the final integraion_test build artifacts to storage. This change moves the arhives destined for GCS to the **C:** drive, then installs the Cloud SDK after they're off of **D:**. The migration of the integration tests only occurs on Windows, for now. Some notes: - We can’t set the build_testapps.py’s --output_directory flag to directly output to the **C:** directory because python’s relpath operations fail when working across different drives. - The **github.workspace** env var can be changed to override check-out & installation paths, but the [GHA team notes potential unknown side effects](https://github.community/t/is-it-possible-to-change-github-workspace/136094/3). Therefore I didn’t test this. - We couldn't use **mv** the results to **C:** as Firestore's build results include symlinks. Using **tar** to ferry the files instead.
1 parent 0172c4d commit 7ae9569

File tree

1 file changed

+54
-17
lines changed

1 file changed

+54
-17
lines changed

.github/workflows/integration_tests.yml

Lines changed: 54 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -253,20 +253,20 @@ jobs:
253253
with:
254254
ref: ${{needs.prepare_matrix.outputs.github_ref}}
255255
submodules: true
256-
- name: Set env vars (ubuntu)
256+
- name: Set env vars (Linux)
257257
if: startsWith(matrix.os, 'ubuntu')
258258
run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV
259-
- name: Set env vars (macos)
259+
- name: Set env vars (macOS)
260260
if: startsWith(matrix.os, 'macos')
261261
run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV
262-
- name: Set env vars (windows)
262+
- name: Set env vars (Windows)
263263
shell: bash
264264
if: startsWith(matrix.os, 'windows')
265265
run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV
266-
- name: Set env vars(all)
266+
- name: Set env vars (all)
267267
shell: bash
268268
run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
269-
- name: Add msbuild to PATH (windows)
269+
- name: Add msbuild to PATH (Windows)
270270
if: startsWith(matrix.os, 'windows')
271271
uses: microsoft/[email protected]
272272
- name: Cache vcpkg C++ dependencies
@@ -293,12 +293,12 @@ jobs:
293293
run: |
294294
pip install -r scripts/gha/requirements.txt
295295
python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
296-
- name: Install OpenSSL (windows)
296+
- name: Install OpenSSL (Windows)
297297
if: matrix.ssl_variant == 'openssl' &&
298298
startsWith(matrix.os, 'windows')
299299
run: |
300300
choco install openssl -r
301-
- name: Install OpenSSL (MacOS)
301+
- name: Install OpenSSL (macOS)
302302
if: matrix.ssl_variant == 'openssl' &&
303303
startsWith(matrix.os, 'macos')
304304
run: |
@@ -341,7 +341,7 @@ jobs:
341341
--artifact_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" \
342342
--noadd_timestamp \
343343
--short_output_paths \
344-
${additional_flags[*]}
344+
${additional_flags[*]}
345345
- name: Upload Desktop integration tests artifact
346346
uses: actions/[email protected]
347347
if: ${{ !cancelled() }}
@@ -356,6 +356,39 @@ jobs:
356356
name: log-artifact
357357
path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}*
358358
retention-days: ${{ env.artifactRetentionDays }}
359+
- name: Cleanup Local Copies of Uploaded Artifacts
360+
shell: bash
361+
run: |
362+
rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}
363+
- name: Configure GCS Upload Directory (Linux, macOS)
364+
if: (startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')) && !cancelled()
365+
shell: bash
366+
run: echo "GCS_UPLOAD_DIR=${{ github.workspace }}" >> $GITHUB_ENV
367+
- name: Configure GCS Upload Directory (Windows)
368+
# For Windows move the uploads to C: to create enough space for the
369+
# Cloud SDK installation on D:.
370+
if: startsWith(matrix.os, 'windows') && !cancelled()
371+
shell: bash
372+
run: echo "GCS_UPLOAD_DIR=/c/fpl-upload" >> $GITHUB_ENV
373+
- name: Move GCS Artifacts to GCS Upload Dir
374+
# Copy the artifacts and also the upload scripts since python's relpath
375+
# produces errors when called across two different drives. Use tar to
376+
# move the files as it neatly deepcopies firestore's symlinks.
377+
if: ${{ (env.GCS_UPLOAD_DIR != github.workspace) && !cancelled() }}
378+
shell: bash
379+
run: |
380+
set -x
381+
mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing
382+
mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted
383+
# The ta directory contains the integration testapps to archive.
384+
tar -chf ${{env.GCS_UPLOAD_DIR}}/ta.tar ta
385+
rm -rf ta
386+
tar xf ${{env.GCS_UPLOAD_DIR}}/ta.tar -C ${{env.GCS_UPLOAD_DIR}}
387+
rm ${{env.GCS_UPLOAD_DIR}}/ta.tar
388+
cp scripts/gha/gcs_uploader.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted/
389+
cp scripts/gha/integration_testing/gcs.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing/
390+
cp scripts/gha/gcs_uploader.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha/
391+
cp scripts/gha-encrypted/gcs_key_file.json ${{env.GCS_UPLOAD_DIR}}//scripts/gha-encrypted/
359392
- name: Set CLOUDSDK_PYTHON (Windows)
360393
shell: bash
361394
if: startsWith(matrix.os, 'windows') && !cancelled()
@@ -364,8 +397,12 @@ jobs:
364397
if: ${{ !cancelled() }}
365398
uses: google-github-actions/setup-gcloud@master
366399
- name: Upload Desktop Artifacts to GCS
400+
shell: bash
367401
if: ${{ !cancelled() }}
368-
run: python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json
402+
run: |
403+
pushd ${{env.GCS_UPLOAD_DIR}}
404+
python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json
405+
popd
369406
- name: Add failure label
370407
# We can mark a failure as soon as any one test fails.
371408
if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
@@ -431,20 +468,20 @@ jobs:
431468
with:
432469
ref: ${{needs.prepare_matrix.outputs.github_ref}}
433470
submodules: true
434-
- name: Set env vars (ubuntu)
471+
- name: Set env vars (Linux)
435472
if: startsWith(matrix.os, 'ubuntu')
436473
run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV
437-
- name: Set env vars (macos)
474+
- name: Set env vars (macOS)
438475
if: startsWith(matrix.os, 'macos')
439476
run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV
440-
- name: Set env vars (windows)
477+
- name: Set env vars (Windows)
441478
shell: bash
442-
if: startsWith(matrix.os, 'windows')
479+
if: startsWith(matrix.os, 'Windows')
443480
run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV
444-
- name: Set env vars(all)
481+
- name: Set env vars (all)
445482
shell: bash
446483
run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
447-
- name: Add msbuild to PATH (windows)
484+
- name: Add msbuild to PATH (Windows)
448485
if: startsWith(matrix.os, 'windows')
449486
uses: microsoft/[email protected]
450487
- name: Cache NDK
@@ -575,9 +612,9 @@ jobs:
575612
with:
576613
ref: ${{needs.prepare_matrix.outputs.github_ref}}
577614
submodules: true
578-
- name: Set env vars (macos)
615+
- name: Set env vars (macOS)
579616
run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV
580-
- name: Set env vars(all)
617+
- name: Set env vars (all)
581618
shell: bash
582619
run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
583620
- name: Setup python

0 commit comments

Comments
 (0)