From a4530c4e8c2b0d70d54af375503a628394b347ec Mon Sep 17 00:00:00 2001 From: Rodolfo Carvalho Date: Sun, 19 Nov 2023 18:11:44 +0100 Subject: [PATCH 1/4] Refactor checking for Git sparse checkout support --- lib/mix/lib/mix/scm/git.ex | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/mix/lib/mix/scm/git.ex b/lib/mix/lib/mix/scm/git.ex index a5471bca189..97a2d878ba2 100644 --- a/lib/mix/lib/mix/scm/git.ex +++ b/lib/mix/lib/mix/scm/git.ex @@ -164,7 +164,7 @@ defmodule Mix.SCM.Git do defp sparse_toggle(opts) do cond do sparse = opts[:sparse] -> - sparse_check(git_version()) + check_sparse_support(git_version()) git!(["--git-dir=.git", "config", "core.sparsecheckout", "true"]) File.mkdir_p!(".git/info") File.write!(".git/info/sparse-checkout", sparse) @@ -180,13 +180,15 @@ defmodule Mix.SCM.Git do end end - defp sparse_check(version) do - unless {1, 7, 4} <= version do - version = version |> Tuple.to_list() |> Enum.join(".") + defp check_sparse_support(version) do + ensure_feature_compatibility(version, {1, 7, 4}, "sparse checkout") + end + defp ensure_feature_compatibility(version, required_version, feature) do + unless required_version <= version do Mix.raise( - "Git >= 1.7.4 is required to use sparse checkout. " <> - "You are running version #{version}" + "Git >= #{format_version(required_version)} is required to use #{feature}. " <> + "You are running version #{format_version(version)}" ) end end @@ -354,6 +356,10 @@ defmodule Mix.SCM.Git do |> List.to_tuple() end + defp format_version(version) do + version |> Tuple.to_list() |> Enum.join(".") + end + defp to_integer(string) do {int, _} = Integer.parse(string) int From 3025d63a10213df50322f78b6282a262bed09d78 Mon Sep 17 00:00:00 2001 From: Rodolfo Carvalho Date: Mon, 20 Nov 2023 11:46:01 +0100 Subject: [PATCH 2/4] Refactor internal references to minimum Git versions --- lib/mix/lib/mix/scm/git.ex | 18 ++++++++++++++---- lib/mix/test/test_helper.exs | 7 ++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/mix/lib/mix/scm/git.ex b/lib/mix/lib/mix/scm/git.ex index 97a2d878ba2..e743bae79ea 100644 --- a/lib/mix/lib/mix/scm/git.ex +++ b/lib/mix/lib/mix/scm/git.ex @@ -180,8 +180,11 @@ defmodule Mix.SCM.Git do end end + @min_git_version_sparse {1, 7, 4} + @min_git_version_progress {1, 7, 1} + defp check_sparse_support(version) do - ensure_feature_compatibility(version, {1, 7, 4}, "sparse checkout") + ensure_feature_compatibility(version, @min_git_version_sparse, "sparse checkout") end defp ensure_feature_compatibility(version, required_version, feature) do @@ -194,7 +197,7 @@ defmodule Mix.SCM.Git do end defp progress_switch(version) do - if {1, 7, 1} <= version, do: ["--progress"], else: [] + if @min_git_version_progress <= version, do: ["--progress"], else: [] end defp tags_switch(nil), do: [] @@ -330,9 +333,16 @@ defmodule Mix.SCM.Git do end end - # Also invoked by lib/mix/test/test_helper.exs + # Invoked by lib/mix/test/test_helper.exs @doc false - def git_version do + def unsupported_options do + git_version = git_version() + + [] + |> Kernel.++(if git_version < @min_git_version_sparse, do: [:sparse], else: []) + end + + defp git_version do case Mix.State.fetch(:git_version) do {:ok, version} -> version diff --git a/lib/mix/test/test_helper.exs b/lib/mix/test/test_helper.exs index 61733a02c5f..8d2734d35c0 100644 --- a/lib/mix/test/test_helper.exs +++ b/lib/mix/test/test_helper.exs @@ -22,7 +22,12 @@ Application.put_env(:logger, :backends, []) os_exclude = if match?({:win32, _}, :os.type()), do: [unix: true], else: [windows: true] epmd_exclude = if match?({:win32, _}, :os.type()), do: [epmd: true], else: [] -git_exclude = if Mix.SCM.Git.git_version() <= {1, 7, 4}, do: [git_sparse: true], else: [] + +git_exclude = + Mix.SCM.Git.unsupported_options() + |> Enum.map(fn + :sparse -> {:git_sparse, true} + end) {line_exclude, line_include} = if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []} From e58f56889eb4437c362ed074b2a97a2a91f67dd4 Mon Sep 17 00:00:00 2001 From: Rodolfo Carvalho Date: Mon, 20 Nov 2023 11:48:18 +0100 Subject: [PATCH 3/4] Refactor Git options validation --- lib/mix/lib/mix/scm/git.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/mix/lib/mix/scm/git.ex b/lib/mix/lib/mix/scm/git.ex index e743bae79ea..5a8eee56dcd 100644 --- a/lib/mix/lib/mix/scm/git.ex +++ b/lib/mix/lib/mix/scm/git.ex @@ -206,6 +206,11 @@ defmodule Mix.SCM.Git do ## Helpers defp validate_git_options(opts) do + opts + |> validate_refspec() + end + + defp validate_refspec(opts) do case Keyword.take(opts, [:branch, :ref, :tag]) do [] -> opts From 849782cd6b4fb0940d3ba97859308f7b6472719b Mon Sep 17 00:00:00 2001 From: Rodolfo Carvalho Date: Sun, 19 Nov 2023 23:58:32 +0100 Subject: [PATCH 4/4] mix deps: Add :depth option to git deps This allows for faster clones that transfer less data over the network and take less space in disk, for cases when the full history is not needed. --- lib/mix/lib/mix/scm/git.ex | 46 +++++- lib/mix/lib/mix/tasks/deps.ex | 4 + lib/mix/test/mix/scm/git_test.exs | 2 +- lib/mix/test/mix/tasks/deps.git_test.exs | 201 +++++++++++++++++++++++ lib/mix/test/test_helper.exs | 1 + 5 files changed, 246 insertions(+), 8 deletions(-) diff --git a/lib/mix/lib/mix/scm/git.ex b/lib/mix/lib/mix/scm/git.ex index 5a8eee56dcd..ea8f1d7eebe 100644 --- a/lib/mix/lib/mix/scm/git.ex +++ b/lib/mix/lib/mix/scm/git.ex @@ -125,14 +125,18 @@ defmodule Mix.SCM.Git do sparse_toggle(opts) update_origin(opts[:git]) + rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts) + # Fetch external data ["--git-dir=.git", "fetch", "--force", "--quiet"] |> Kernel.++(progress_switch(git_version())) |> Kernel.++(tags_switch(opts[:tag])) + |> Kernel.++(depth_switch(opts[:depth])) + |> Kernel.++(if rev, do: ["origin", rev], else: []) |> git!() # Migrate the Git repo - rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts) || default_branch() + rev = rev || default_branch() git!(["--git-dir=.git", "checkout", "--quiet", rev]) if opts[:submodules] do @@ -181,12 +185,17 @@ defmodule Mix.SCM.Git do end @min_git_version_sparse {1, 7, 4} + @min_git_version_depth {1, 5, 0} @min_git_version_progress {1, 7, 1} defp check_sparse_support(version) do ensure_feature_compatibility(version, @min_git_version_sparse, "sparse checkout") end + defp check_depth_support(version) do + ensure_feature_compatibility(version, @min_git_version_depth, "depth (shallow clone)") + end + defp ensure_feature_compatibility(version, required_version, feature) do unless required_version <= version do Mix.raise( @@ -203,11 +212,19 @@ defmodule Mix.SCM.Git do defp tags_switch(nil), do: [] defp tags_switch(_), do: ["--tags"] + defp depth_switch(nil), do: [] + + defp depth_switch(n) when is_integer(n) and n > 0 do + check_depth_support(git_version()) + ["--depth=#{n}"] + end + ## Helpers defp validate_git_options(opts) do opts |> validate_refspec() + |> validate_depth() end defp validate_refspec(opts) do @@ -232,6 +249,22 @@ defmodule Mix.SCM.Git do end end + defp validate_depth(opts) do + case Keyword.take(opts, [:depth]) do + [] -> + opts + + [{:depth, depth}] when is_integer(depth) and depth > 0 -> + opts + + invalid_depth -> + Mix.raise( + "The depth must be a positive integer, and be specified only once, got: #{inspect(invalid_depth)}. " <> + "Error on Git dependency: #{redact_uri(opts[:git])}" + ) + end + end + defp get_lock(opts) do %{rev: rev} = get_rev_info() {:git, opts[:git], rev, get_lock_opts(opts)} @@ -248,7 +281,7 @@ defmodule Mix.SCM.Git do defp get_lock_rev(_, _), do: nil defp get_lock_opts(opts) do - lock_opts = Keyword.take(opts, [:branch, :ref, :tag, :sparse, :subdir]) + lock_opts = Keyword.take(opts, [:branch, :ref, :tag, :sparse, :subdir, :depth]) if opts[:submodules] do lock_opts ++ [submodules: true] @@ -258,11 +291,7 @@ defmodule Mix.SCM.Git do end defp get_opts_rev(opts) do - if branch = opts[:branch] do - "origin/#{branch}" - else - opts[:ref] || opts[:tag] - end + opts[:branch] || opts[:ref] || opts[:tag] end defp redact_uri(git) do @@ -292,6 +321,8 @@ defmodule Mix.SCM.Git do end defp default_branch() do + # Note: the `set-head -a` command requires the remote reference to be + # fetched first. git!(["--git-dir=.git", "remote", "set-head", "origin", "-a"]) "origin/HEAD" end @@ -345,6 +376,7 @@ defmodule Mix.SCM.Git do [] |> Kernel.++(if git_version < @min_git_version_sparse, do: [:sparse], else: []) + |> Kernel.++(if git_version < @min_git_version_depth, do: [:depth], else: []) end defp git_version do diff --git a/lib/mix/lib/mix/tasks/deps.ex b/lib/mix/lib/mix/tasks/deps.ex index ff51f089dcf..d1acb08db5c 100644 --- a/lib/mix/lib/mix/tasks/deps.ex +++ b/lib/mix/lib/mix/tasks/deps.ex @@ -120,6 +120,10 @@ defmodule Mix.Tasks.Deps do * `:subdir` - (since v1.13.0) search for the project in the given directory relative to the git checkout. This is similar to `:sparse` option but instead of a doing a sparse checkout it does a full checkout. + * `:depth` - (since v1.17.0) creates a shallow clone of the Git repository, + limiting the history to the specified number of commits. This can significantly + improve clone speed for large repositories when full history is not needed. + The value must be a positive integer, typically `1`. If your Git repository requires authentication, such as basic username:password HTTP authentication via URLs, it can be achieved via Git configuration, keeping diff --git a/lib/mix/test/mix/scm/git_test.exs b/lib/mix/test/mix/scm/git_test.exs index 24429510544..d572ba06f0c 100644 --- a/lib/mix/test/mix/scm/git_test.exs +++ b/lib/mix/test/mix/scm/git_test.exs @@ -32,7 +32,7 @@ defmodule Mix.SCM.GitTest do "https://github.com/elixir-lang/some_dep.git - v1" assert Mix.SCM.Git.format(Keyword.put(opts, :branch, "b")) == - "https://github.com/elixir-lang/some_dep.git - origin/b" + "https://github.com/elixir-lang/some_dep.git - b" assert Mix.SCM.Git.format(Keyword.put(opts, :ref, "abcdef")) == "https://github.com/elixir-lang/some_dep.git - abcdef" diff --git a/lib/mix/test/mix/tasks/deps.git_test.exs b/lib/mix/test/mix/tasks/deps.git_test.exs index 8983a6abb73..a5338b2db3e 100644 --- a/lib/mix/test/mix/tasks/deps.git_test.exs +++ b/lib/mix/test/mix/tasks/deps.git_test.exs @@ -478,6 +478,207 @@ defmodule Mix.Tasks.DepsGitTest do purge([GitRepo, GitRepo.MixProject]) end + describe "Git depth option" do + @describetag :git_depth + + test "gets and updates Git repos with depth option" do + Process.put(:git_repo_opts, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + # Expand depth + update_dep(depth: 2) + Mix.Tasks.Deps.Get.run([]) + assert_shallow("deps/git_repo", 2) + + # Reduce depth + update_dep(depth: 1) + Mix.Tasks.Deps.Get.run([]) + assert_shallow("deps/git_repo", 1) + end) + end + + test "with tag" do + Process.put(:git_repo_opts, depth: 1, tag: "with_module") + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - with_module)" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + end) + end + + test "with branch" do + Process.put(:git_repo_opts, depth: 1, branch: "main") + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - main)" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + end) + end + + test "with ref" do + [last, _ | _] = get_git_repo_revs("git_repo") + + Process.put(:git_repo_opts, depth: 1, ref: last) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - #{last})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + end) + end + + test "changing refspec updates retaining depth" do + [last, first | _] = get_git_repo_revs("git_repo") + + Process.put(:git_repo_opts, ref: first, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - #{first})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + assert File.read!("mix.lock") =~ first + + # Change refspec + update_dep(ref: last, depth: 1) + Mix.Tasks.Deps.Get.run([]) + assert_shallow("deps/git_repo", 1) + assert File.read!("mix.lock") =~ last + end) + end + + test "removing depth retains shallow repository" do + # For compatibility and simplicity, we follow Git's behavior and do not + # attempt to unshallow an existing repository. This should not be a + # problem, because all we guarantee is that the correct source code is + # available whenever mix.exs or mix.lock change. If one wanted to have a + # full clone, they can always run `deps.clean` and `deps.get` again. + Process.put(:git_repo_opts, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + # Remove depth + update_dep([]) + Mix.Tasks.Deps.Get.run([]) + refute File.read!("mix.lock") =~ "depth:" + assert File.exists?("deps/git_repo/.git/shallow") + + assert System.cmd("git", ~w[--git-dir=deps/git_repo/.git rev-list --count HEAD]) == + {"1\n", 0} + end) + end + + @tag :git_sparse + test "with sparse checkout" do + Process.put(:git_repo_opts, sparse: "sparse_dir", depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + refute File.exists?("deps/git_repo/mix.exs") + assert File.exists?("deps/git_repo/sparse_dir/mix.exs") + assert File.read!("mix.lock") =~ "sparse: \"sparse_dir\"" + end) + end + + test "with subdir" do + Process.put(:git_repo_opts, subdir: "sparse_dir", depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + assert File.exists?("deps/git_repo/mix.exs") + assert File.exists?("deps/git_repo/sparse_dir/mix.exs") + assert File.read!("mix.lock") =~ "subdir: \"sparse_dir\"" + end) + end + + test "does not affect submodules depth" do + # The expectation is that we can add an explicit option in the future, + # just like git-clone has `--shallow-submodules`. + Process.put(:git_repo_opts, submodules: true, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + assert File.read!("mix.lock") =~ "submodules: true" + # TODO: assert submodule is not shallow. This would likely require + # changes to the fixtures. Apparently, not even the submodules-specific + # tests check that the cloned repo contains submodules as expected. + end) + end + + defp update_dep(git_repo_opts) do + # Flush the errors we got, move to a clean slate + Mix.shell().flush() + Mix.Task.clear() + Process.put(:git_repo_opts, git_repo_opts) + Mix.Project.pop() + Mix.Project.push(GitApp) + end + + defp assert_shallow(repo_path, depth) do + assert File.read!("mix.lock") =~ "depth: #{depth}" + + # Check if the repository is a shallow clone + assert File.exists?(repo_path <> "/.git/shallow") + + # Check the number of commits in the current branch. + # + # We could consider all branches with `git rev-list --count --all`, as in + # practice there should be only a single branch. However, the test fixture + # sets up two branches, and that brings us to an interesting situation: + # instead of guaranteeing that the `:depth` option would keep the + # repository lean even after refspec changes, we only guarantee the number + # of commits in the current branch, perhaps leaving more objects around + # than strictly necessary. This allows us to keep the implementation + # simple, while still providing a reasonable guarantee. + assert System.cmd("git", ~w[--git-dir=#{repo_path}/.git rev-list --count HEAD]) == + {"#{depth}\n", 0} + end + end + defp refresh(post_config) do %{name: name, file: file} = Mix.Project.pop() Mix.ProjectStack.post_config(post_config) diff --git a/lib/mix/test/test_helper.exs b/lib/mix/test/test_helper.exs index 8d2734d35c0..c88194a04cb 100644 --- a/lib/mix/test/test_helper.exs +++ b/lib/mix/test/test_helper.exs @@ -27,6 +27,7 @@ git_exclude = Mix.SCM.Git.unsupported_options() |> Enum.map(fn :sparse -> {:git_sparse, true} + :depth -> {:git_depth, true} end) {line_exclude, line_include} =