Skip to content

Commit d54f106

Browse files
SerjleeMatteo-it
Serjlee
and
Matteo-it
authored
Split CI build, restore Cloudflare purge for datasheets (#2153)
* Split CI build * Skip everything on cache hit * Fix output path * Move datasheets rendering to own action file * Set name * Add debug step for datasheets * Update production workflow, too * Fix APP_ENV * Checkout is inevitable * Fix purge * Explicit checkout * Typo * Remove restore-key * Use correct folder * 2.0.18-2-alpha.1 * 2.0.18-2 * Fail on purge fail, debug prefix and zone * Remove debug log --------- Co-authored-by: Matteo Marsala <[email protected]>
1 parent ad0c538 commit d54f106

File tree

8 files changed

+210
-78
lines changed

8 files changed

+210
-78
lines changed

‎.github/actions/cleanup-disk/action.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Disk cleanup
2+
description: "Cleanup disk space"
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Free up disk space
7+
shell: bash
8+
run: |
9+
echo "Disk space before cleanup..."
10+
df -h /
11+
echo "Removing unnecessary files to free up disk space..."
12+
# https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832
13+
sudo rm -rf \
14+
/opt/hostedtoolcache \
15+
/opt/google/chrome \
16+
/opt/microsoft/msedge \
17+
/opt/microsoft/powershell \
18+
/opt/pipx \
19+
/usr/lib/mono \
20+
/usr/local/julia* \
21+
/usr/local/lib/android \
22+
/usr/local/lib/node_modules \
23+
/usr/local/share/chromium \
24+
/usr/local/share/powershell \
25+
/usr/share/dotnet \
26+
/usr/share/swift
27+
echo "Disk space after cleanup..."
28+
df -h /

‎.github/actions/cloudflare-upload/action.yml

-3
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ inputs:
2020
runs:
2121
using: composite
2222
steps:
23-
2423
- name: Find PR Preview Comment
2524
if: github.event_name == 'pull_request'
2625
uses: peter-evans/find-comment@v1
@@ -67,7 +66,6 @@ runs:
6766
🚀 Preview this PR: ${{ steps.deploy-cloudflare.outputs.url }}
6867
📍 Commit SHA: ${{ github.sha }}
6968
70-
7169
- name: Update PR Preview Comment
7270
if: github.event_name == 'pull_request' && steps.deploy-preview-comment.outputs.comment-id != 0
7371
uses: peter-evans/[email protected]
@@ -78,4 +76,3 @@ runs:
7876
### ${{ inputs.project-name }}
7977
🚀 Preview this PR: ${{ steps.deploy-cloudflare.outputs.url }}
8078
📍 Commit SHA: ${{ github.sha }}
81-

‎.github/actions/generate-datasheets/action.yml

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: "Generate Datasheets"
2+
description: "Generate product datasheets from markdown files"
3+
inputs:
4+
datasheets-path:
5+
description: "The datasheets path"
6+
required: true
7+
default: static/resources/datasheets
8+
artifact-name:
9+
description: "The name of the output artifact"
10+
required: true
11+
12+
runs:
13+
using: composite
14+
steps:
15+
- uses: actions/cache@v4
16+
id: cache
17+
with:
18+
path: ${{ inputs.datasheets-path }}
19+
key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }}
20+
21+
- uses: actions/setup-node@v4
22+
if: steps.cache.outputs.cache-hit != 'true'
23+
with:
24+
node-version: 18
25+
cache: "npm"
26+
cache-dependency-path: "package-lock.json"
27+
28+
- name: Render Datasheets
29+
if: steps.cache.outputs.cache-hit != 'true'
30+
run: |
31+
cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering
32+
./render-datasheets.sh
33+
cd $GITHUB_WORKSPACE
34+
mkdir -p ${{ inputs.datasheets-path }}
35+
find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./${{ inputs.datasheets-path }}/ \;
36+
shell: bash
37+
38+
- name: Export artifact
39+
uses: actions/upload-artifact@v4
40+
with:
41+
name: ${{ inputs.artifact-name }}
42+
path: ${{ inputs.datasheets-path }}
43+
retention-days: 1 # Only needed to pass it to the next job

‎.github/actions/sync-s3/action.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Sync assets on S3
2+
description: "Sync Docs assets on S3"
3+
inputs:
4+
role-to-assume:
5+
description: "The IAM role to assume"
6+
required: true
7+
bucket-name:
8+
description: "The name of the S3 bucket to sync assets to"
9+
required: true
10+
runs:
11+
using: composite
12+
steps:
13+
- name: Configure AWS credentials from Staging account
14+
uses: aws-actions/configure-aws-credentials@v4
15+
with:
16+
role-to-assume: ${{ inputs.role-to-assume }}
17+
aws-region: us-east-1
18+
19+
- name: Sync all cacheable assets
20+
shell: bash
21+
run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ inputs.bucket-name }}/
22+
23+
- name: Sync all non-cacheable assets
24+
shell: bash
25+
# Don't cache any HTML or JSON file: they should always be up-to-dates
26+
run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ inputs.bucket-name }}/
27+
28+
- name: Sync PDF
29+
shell: bash
30+
run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ inputs.bucket-name }}/

‎.github/workflows/deploy-production.yml

+52-35
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,30 @@ concurrency:
1010
group: deploy-production
1111
cancel-in-progress: true
1212

13+
# Allow installation of dependencies
1314
permissions:
1415
id-token: write
1516
contents: read
1617

1718
jobs:
19+
# This job is used to render datasheets, but only if they have changed.
20+
# It's a separate job so we don't have to cleanup the machine afterwards.
21+
render-datasheets:
22+
name: Render Datasheets
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v4
26+
with:
27+
fetch-depth: 1
28+
29+
- uses: ./.github/actions/generate-datasheets
30+
with:
31+
artifact-name: datasheets
32+
datasheets-path: static/resources/datasheets
33+
1834
build:
19-
if: "github.repository == 'arduino/docs-content'"
35+
name: Build and Deploy
36+
needs: render-datasheets
2037
runs-on: ubuntu-latest
2138
environment: production
2239
env:
@@ -27,20 +44,28 @@ jobs:
2744
- uses: actions/checkout@v4
2845
with:
2946
fetch-depth: 1
47+
48+
- name: Cleanup runner disk
49+
uses: ./.github/actions/cleanup-disk # Cleanup machine before starting the build
50+
3051
- uses: actions/setup-node@v4
3152
with:
3253
node-version: 18
3354
cache: "npm"
34-
cache-dependency-path: "**/package-lock.json"
55+
cache-dependency-path: "package-lock.json"
56+
57+
- name: Retrieve Datasheets
58+
uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job
59+
with:
60+
name: datasheets
61+
path: static/resources/datasheets
3562

36-
- name: Render Datasheets
37-
run: cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh
63+
- name: Debug datasheet list
64+
run: ls -lah static/resources/datasheets
3865

3966
- name: Copy Static Files
4067
run: |
41-
mkdir -p static/resources/datasheets static/resources/schematics static/resources/pinouts static/resources/models
42-
find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \;
43-
find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \;
68+
mkdir -p static/resources/schematics static/resources/pinouts static/resources/models
4469
find ./content/hardware -type f -name "*-full-pinout.pdf" -exec cp {} ./static/resources/pinouts/ \;
4570
find ./content/hardware -type f -name "*-pinout.png" -exec cp {} ./static/resources/pinouts/ \;
4671
find ./content/hardware -type f -name "*-step.zip" -exec cp {} ./static/resources/models/ \;
@@ -54,7 +79,7 @@ jobs:
5479
restore-keys: |
5580
${{ runner.os }}-cache-gatsby-main
5681
57-
- name: Gatsby Public Folder
82+
- name: Gatsby Public Folder cache
5883
uses: actions/cache@v4
5984
id: gatsby-public-folder
6085
with:
@@ -64,37 +89,29 @@ jobs:
6489
${{ runner.os }}-public-gatsby-main
6590
6691
- run: npm install
92+
6793
- run: npm run build
6894

69-
- name: Clean up node_modules
95+
- name: Clean up node_modules # Just to save space
7096
run: rm -rf node_modules
7197

72-
- name: Configure AWS credentials from Production account
73-
uses: aws-actions/configure-aws-credentials@v4
98+
- name: Deploy to S3
99+
uses: ./.github/actions/sync-s3
74100
with:
75101
role-to-assume: ${{ secrets.PRODUCTION_IAM_ROLE }}
76-
aws-region: us-east-1
77-
78-
- name: Sync all cacheable assets
79-
run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/
80-
81-
- name: Sync all non-cacheable assets
82-
# Don't cache any HTML or JSON file: they should always be up-to-dates
83-
run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/
84-
85-
# - name: Sync PDF
86-
# run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/
87-
88-
# - name: Purge cache on CloudFlare
89-
# run: |
90-
# curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \
91-
# -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \
92-
# -H "Content-Type: application/json" \
93-
# --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}'
102+
bucket-name: ${{ secrets.PRODUCTION_BUCKET_NAME }}
94103

95-
- name: Sync all cacheable assets
96-
run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/
97-
98-
- name: Sync all non-cacheable assets
99-
# Don't cache any HTML or JSON file: they should always be up-to-dates
100-
run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/
104+
purge-datasheets:
105+
name: Purge Datasheets cache
106+
needs: build
107+
runs-on: ubuntu-latest
108+
environment: production
109+
steps:
110+
- name: Purge Cloudflare Cache
111+
shell: bash
112+
run: |
113+
echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ vars.CLOUDFLARE_ZONE }}"
114+
curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \
115+
-H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \
116+
-H "Content-Type: application/json" \
117+
--data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}'

‎.github/workflows/deploy-staging.yml

+52-35
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,30 @@ concurrency:
1010
group: deploy-staging
1111
cancel-in-progress: true
1212

13+
# Allow installation of dependencies
1314
permissions:
1415
id-token: write
1516
contents: read
1617

1718
jobs:
19+
# This job is used to render datasheets, but only if they have changed.
20+
# It's a separate job so we don't have to cleanup the machine afterwards.
21+
render-datasheets:
22+
name: Render Datasheets
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v4
26+
with:
27+
fetch-depth: 1
28+
29+
- uses: ./.github/actions/generate-datasheets
30+
with:
31+
artifact-name: datasheets
32+
datasheets-path: static/resources/datasheets
33+
1834
build:
35+
name: Build and Deploy
36+
needs: render-datasheets
1937
runs-on: ubuntu-latest
2038
environment: staging
2139
env:
@@ -26,20 +44,28 @@ jobs:
2644
- uses: actions/checkout@v4
2745
with:
2846
fetch-depth: 1
47+
48+
- name: Cleanup runner disk
49+
uses: ./.github/actions/cleanup-disk # Cleanup machine before starting the build
50+
2951
- uses: actions/setup-node@v4
3052
with:
3153
node-version: 18
3254
cache: "npm"
33-
cache-dependency-path: "**/package-lock.json"
55+
cache-dependency-path: "package-lock.json"
56+
57+
- name: Retrieve Datasheets
58+
uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job
59+
with:
60+
name: datasheets
61+
path: static/resources/datasheets
3462

35-
- name: Render Datasheets
36-
run: cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh
63+
- name: Debug datasheet list
64+
run: ls -lah static/resources/datasheets
3765

3866
- name: Copy Static Files
3967
run: |
40-
mkdir -p static/resources/datasheets static/resources/schematics static/resources/pinouts static/resources/models
41-
find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \;
42-
find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \;
68+
mkdir -p static/resources/schematics static/resources/pinouts static/resources/models
4369
find ./content/hardware -type f -name "*-full-pinout.pdf" -exec cp {} ./static/resources/pinouts/ \;
4470
find ./content/hardware -type f -name "*-pinout.png" -exec cp {} ./static/resources/pinouts/ \;
4571
find ./content/hardware -type f -name "*-step.zip" -exec cp {} ./static/resources/models/ \;
@@ -53,7 +79,7 @@ jobs:
5379
restore-keys: |
5480
${{ runner.os }}-cache-gatsby-main
5581
56-
- name: Gatsby Public Folder
82+
- name: Gatsby Public Folder cache
5783
uses: actions/cache@v4
5884
id: gatsby-public-folder
5985
with:
@@ -63,38 +89,29 @@ jobs:
6389
${{ runner.os }}-public-gatsby-main
6490
6591
- run: npm install
92+
6693
- run: npm run build
6794

68-
- name: Clean up node_modules
95+
- name: Clean up node_modules # Just to save space
6996
run: rm -rf node_modules
7097

71-
- name: Configure AWS credentials from Staging account
72-
uses: aws-actions/configure-aws-credentials@v4
98+
- name: Deploy to S3
99+
uses: ./.github/actions/sync-s3
73100
with:
74101
role-to-assume: ${{ secrets.STAGING_IAM_ROLE }}
75-
aws-region: us-east-1
76-
77-
# - name: Sync all cacheable assets
78-
# run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/
79-
80-
# - name: Sync all non-cacheable assets
81-
# # Don't cache any HTML or JSON file: they should always be up-to-dates
82-
# run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/
83-
84-
# - name: Sync PDF
85-
# run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/
86-
87-
# - name: Purge cache on CloudFlare
88-
# run: |
89-
# curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \
90-
# -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \
91-
# -H "Content-Type: application/json" \
92-
# --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}'
93-
94-
- name: Sync all cacheable assets
95-
run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/
96-
97-
- name: Sync all non-cacheable assets
98-
# Don't cache any HTML or JSON file: they should always be up-to-dates
99-
run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/
102+
bucket-name: ${{ secrets.STAGING_BUCKET_NAME }}
100103

104+
purge-datasheets:
105+
name: Purge Datasheets cache
106+
needs: build
107+
runs-on: ubuntu-latest
108+
environment: staging
109+
steps:
110+
- name: Purge Cloudflare Cache
111+
shell: bash
112+
run: |
113+
echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ vars.CLOUDFLARE_ZONE }}"
114+
curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \
115+
-H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \
116+
-H "Content-Type: application/json" \
117+
--data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}'

0 commit comments

Comments
 (0)