diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 72d441c4e..38064543f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -4,6 +4,8 @@ on: [push, pull_request]
 env:
   RUSTDOCFLAGS: -Dwarnings
   RUSTFLAGS: -Dwarnings
+  RUST_LLVM_VERSION: 18.0-2024-02-13
+  RUST_COMPILER_RT_ROOT: ./compiler-rt
 
 jobs:
   test:
@@ -105,11 +107,28 @@ jobs:
       shell: bash
     - run: rustup target add ${{ matrix.target }}
     - run: rustup component add llvm-tools-preview
+    - uses: Swatinem/rust-cache@v2
+      with:
+        key: ${{ matrix.target }}
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      if: matrix.os == 'ubuntu-latest'
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ matrix.target }}-buildx-${{ github.sha }}
+        restore-keys: ${{ matrix.target }}-buildx-
+        
+    - name: Cache compiler-rt
+      id: cache-compiler-rt
+      uses: actions/cache@v4
+      with:
+        path: compiler-rt
+        key: ${{ runner.os }}-compiler-rt-${{ env.RUST_LLVM_VERSION }}
     - name: Download compiler-rt reference sources
+      if: steps.cache-compiler-rt.outputs.cache-hit != 'true'
       run: |
-        curl -L -o code.tar.gz https://github.com/rust-lang/llvm-project/archive/rustc/18.0-2024-02-13.tar.gz
-        tar xzf code.tar.gz --strip-components 1 llvm-project-rustc-18.0-2024-02-13/compiler-rt
-        echo RUST_COMPILER_RT_ROOT=./compiler-rt >> $GITHUB_ENV
+        curl -L -o code.tar.gz "https://github.com/rust-lang/llvm-project/archive/rustc/${RUST_LLVM_VERSION}.tar.gz"
+        tar xzf code.tar.gz --strip-components 1 llvm-project-rustc-${RUST_LLVM_VERSION}/compiler-rt
       shell: bash
 
     # Non-linux tests just use our raw script
@@ -120,10 +139,23 @@ jobs:
         NO_STD: ${{ matrix.no_std }}
         TEST_VERBATIM: ${{ matrix.test_verbatim }}
 
+    # Configure buildx to use Docker layer caching
+    - uses: docker/setup-buildx-action@v3
+      if: matrix.os == 'ubuntu-latest'
+
     # Otherwise we use our docker containers to run builds
     - run: cargo generate-lockfile && ./ci/run-docker.sh ${{ matrix.target }}
       if: matrix.os == 'ubuntu-latest'
 
+    # Workaround to keep Docker cache smaller
+    # https://github.com/docker/build-push-action/issues/252
+    # https://github.com/moby/buildkit/issues/1896
+    - name: Move Docker cache
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        rm -rf /tmp/.buildx-cache
+        mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
   rustfmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -147,6 +179,7 @@ jobs:
     - name: Install nightly `clippy`
       run: |
         rustup set profile minimal && rustup default "nightly-$(curl -s https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu/clippy)" && rustup component add clippy
+    - uses: Swatinem/rust-cache@v2
     - run: cargo clippy -- -D clippy::all
 
   success:
diff --git a/ci/run-docker.sh b/ci/run-docker.sh
index e5ff8a46b..14b1a32d7 100755
--- a/ci/run-docker.sh
+++ b/ci/run-docker.sh
@@ -41,7 +41,19 @@ run() {
       export RUST_COMPILER_RT_ROOT=./compiler-rt
     fi
 
-    docker build \
+    if [ "$GITHUB_ACTIONS" = "true" ]; then
+      # Enable Docker image caching on GHA
+      
+      buildx="buildx"
+      build_args=(
+        "--cache-from" "type=local,src=/tmp/.buildx-cache"
+        "--cache-to" "type=local,dest=/tmp/.buildx-cache-new"
+        "${build_args[@]:-}"
+        "--load"
+      )
+    fi
+
+    docker "${buildx:-}" build \
            -t "builtins-$target" \
            ${build_args[@]:-} \
            "ci/docker/$target"