Auto merge of rust-lang#98439 - ehuss:cleanup-ci-script, r=Mark-Simulacrum

Clean up submodule checkout scripts

This is just some small cleanup:

* Removed unused CACHE_DIR stuff
* Removed duplicate fetch_github_commit_archive function which is no longer used
* Combined init_repo.sh and checkout-submodules.sh, as checkout-submodules.sh was doing nothing but calling init_repo.sh

Author: bors

src/ci/docker/host-x86_64/dist-various-2/shared.sh

@@ -33,15 +33,3 @@ function retry {
     }
   done
 }
-
-# Copied from ../../init_repo.sh
-function fetch_github_commit_archive {
-    local module=$1
-    local cached="download-${module//\//-}.tar.gz"
-    retry sh -c "rm -f $cached && \
-        curl -f -sSL -o $cached $2"
-    mkdir $module
-    touch "$module/.git"
-    tar -C $module --strip-components=1 -xf $cached
-    rm $cached
-}

src/ci/init_repo.sh

@@ -2,16 +2,70 @@
 # Check out all our submodules, but more quickly than using git by using one of
 # our custom scripts
 
-set -euo pipefail
-IFS=$'\n\t'
+set -o errexit
+set -o pipefail
+set -o nounset
 
-source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
+if [ ! -d ".git" ]; then
+    echo "Error: This must run in the root of the repository"
+    exit 1
+fi
+
+ci_dir=$(cd $(dirname $0) && pwd)/..
+. "$ci_dir/shared.sh"
 
-if isWindows; then
-    path="/c/cache/rustsrc"
-else
-    path="${HOME}/rustsrc"
+# On the beta channel we'll be automatically calculating the prerelease version
+# via the git history, so unshallow our shallow clone from CI.
+if [ "$(releaseChannel)" = "beta" ]; then
+  git fetch origin --unshallow beta master
 fi
 
-mkdir -p "${path}"
-"$(cd "$(dirname "$0")" && pwd)/../init_repo.sh" . "${path}"
+function fetch_github_commit_archive {
+    local module=$1
+    local cached="download-${module//\//-}.tar.gz"
+    retry sh -c "rm -f $cached && \
+        curl -f -sSL -o $cached $2"
+    mkdir $module
+    touch "$module/.git"
+    # On Windows, the default behavior is to emulate symlinks by copying
+    # files. However, that ends up being order-dependent while extracting,
+    # which can cause a failure if the symlink comes first. This env var
+    # causes tar to use real symlinks instead, which are allowed to dangle.
+    export MSYS=winsymlinks:nativestrict
+    tar -C $module --strip-components=1 -xf $cached
+    rm $cached
+}
+
+# Archive downloads are temporarily disabled due to sudden 504
+# gateway timeout errors.
+# included="src/llvm-project src/doc/book src/doc/rust-by-example"
+included=""
+modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)"
+modules=($modules)
+use_git=""
+urls="$(git config --file .gitmodules --get-regexp '\.url$' | cut -d' ' -f2)"
+urls=($urls)
+# shellcheck disable=SC2068
+for i in ${!modules[@]}; do
+    module=${modules[$i]}
+    if [[ " $included " = *" $module "* ]]; then
+        commit="$(git ls-tree HEAD $module | awk '{print $3}')"
+        git rm $module
+        url=${urls[$i]}
+        url=${url/\.git/}
+        fetch_github_commit_archive $module "$url/archive/$commit.tar.gz" &
+        bg_pids[${i}]=$!
+        continue
+    else
+        use_git="$use_git $module"
+    fi
+done
+retry sh -c "git submodule deinit -f $use_git && \
+    git submodule sync && \
+    git submodule update -j 16 --init --recursive --depth 1 $use_git"
+# STATUS=0
+# for pid in ${bg_pids[*]}
+# do
+#     wait $pid || STATUS=1
+# done
+# exit ${STATUS}