diff --git a/lib/mix/lib/mix/scm/git.ex b/lib/mix/lib/mix/scm/git.ex index a5471bca189..ea8f1d7eebe 100644 --- a/lib/mix/lib/mix/scm/git.ex +++ b/lib/mix/lib/mix/scm/git.ex @@ -125,14 +125,18 @@ defmodule Mix.SCM.Git do sparse_toggle(opts) update_origin(opts[:git]) + rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts) + # Fetch external data ["--git-dir=.git", "fetch", "--force", "--quiet"] |> Kernel.++(progress_switch(git_version())) |> Kernel.++(tags_switch(opts[:tag])) + |> Kernel.++(depth_switch(opts[:depth])) + |> Kernel.++(if rev, do: ["origin", rev], else: []) |> git!() # Migrate the Git repo - rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts) || default_branch() + rev = rev || default_branch() git!(["--git-dir=.git", "checkout", "--quiet", rev]) if opts[:submodules] do @@ -164,7 +168,7 @@ defmodule Mix.SCM.Git do defp sparse_toggle(opts) do cond do sparse = opts[:sparse] -> - sparse_check(git_version()) + check_sparse_support(git_version()) git!(["--git-dir=.git", "config", "core.sparsecheckout", "true"]) File.mkdir_p!(".git/info") File.write!(".git/info/sparse-checkout", sparse) @@ -180,27 +184,50 @@ defmodule Mix.SCM.Git do end end - defp sparse_check(version) do - unless {1, 7, 4} <= version do - version = version |> Tuple.to_list() |> Enum.join(".") + @min_git_version_sparse {1, 7, 4} + @min_git_version_depth {1, 5, 0} + @min_git_version_progress {1, 7, 1} + + defp check_sparse_support(version) do + ensure_feature_compatibility(version, @min_git_version_sparse, "sparse checkout") + end + + defp check_depth_support(version) do + ensure_feature_compatibility(version, @min_git_version_depth, "depth (shallow clone)") + end + defp ensure_feature_compatibility(version, required_version, feature) do + unless required_version <= version do Mix.raise( - "Git >= 1.7.4 is required to use sparse checkout. " <> - "You are running version #{version}" + "Git >= #{format_version(required_version)} is required to use #{feature}. " <> + "You are running version #{format_version(version)}" ) end end defp progress_switch(version) do - if {1, 7, 1} <= version, do: ["--progress"], else: [] + if @min_git_version_progress <= version, do: ["--progress"], else: [] end defp tags_switch(nil), do: [] defp tags_switch(_), do: ["--tags"] + defp depth_switch(nil), do: [] + + defp depth_switch(n) when is_integer(n) and n > 0 do + check_depth_support(git_version()) + ["--depth=#{n}"] + end + ## Helpers defp validate_git_options(opts) do + opts + |> validate_refspec() + |> validate_depth() + end + + defp validate_refspec(opts) do case Keyword.take(opts, [:branch, :ref, :tag]) do [] -> opts @@ -222,6 +249,22 @@ defmodule Mix.SCM.Git do end end + defp validate_depth(opts) do + case Keyword.take(opts, [:depth]) do + [] -> + opts + + [{:depth, depth}] when is_integer(depth) and depth > 0 -> + opts + + invalid_depth -> + Mix.raise( + "The depth must be a positive integer, and be specified only once, got: #{inspect(invalid_depth)}. " <> + "Error on Git dependency: #{redact_uri(opts[:git])}" + ) + end + end + defp get_lock(opts) do %{rev: rev} = get_rev_info() {:git, opts[:git], rev, get_lock_opts(opts)} @@ -238,7 +281,7 @@ defmodule Mix.SCM.Git do defp get_lock_rev(_, _), do: nil defp get_lock_opts(opts) do - lock_opts = Keyword.take(opts, [:branch, :ref, :tag, :sparse, :subdir]) + lock_opts = Keyword.take(opts, [:branch, :ref, :tag, :sparse, :subdir, :depth]) if opts[:submodules] do lock_opts ++ [submodules: true] @@ -248,11 +291,7 @@ defmodule Mix.SCM.Git do end defp get_opts_rev(opts) do - if branch = opts[:branch] do - "origin/#{branch}" - else - opts[:ref] || opts[:tag] - end + opts[:branch] || opts[:ref] || opts[:tag] end defp redact_uri(git) do @@ -282,6 +321,8 @@ defmodule Mix.SCM.Git do end defp default_branch() do + # Note: the `set-head -a` command requires the remote reference to be + # fetched first. git!(["--git-dir=.git", "remote", "set-head", "origin", "-a"]) "origin/HEAD" end @@ -328,9 +369,17 @@ defmodule Mix.SCM.Git do end end - # Also invoked by lib/mix/test/test_helper.exs + # Invoked by lib/mix/test/test_helper.exs @doc false - def git_version do + def unsupported_options do + git_version = git_version() + + [] + |> Kernel.++(if git_version < @min_git_version_sparse, do: [:sparse], else: []) + |> Kernel.++(if git_version < @min_git_version_depth, do: [:depth], else: []) + end + + defp git_version do case Mix.State.fetch(:git_version) do {:ok, version} -> version @@ -354,6 +403,10 @@ defmodule Mix.SCM.Git do |> List.to_tuple() end + defp format_version(version) do + version |> Tuple.to_list() |> Enum.join(".") + end + defp to_integer(string) do {int, _} = Integer.parse(string) int diff --git a/lib/mix/lib/mix/tasks/deps.ex b/lib/mix/lib/mix/tasks/deps.ex index ff51f089dcf..d1acb08db5c 100644 --- a/lib/mix/lib/mix/tasks/deps.ex +++ b/lib/mix/lib/mix/tasks/deps.ex @@ -120,6 +120,10 @@ defmodule Mix.Tasks.Deps do * `:subdir` - (since v1.13.0) search for the project in the given directory relative to the git checkout. This is similar to `:sparse` option but instead of a doing a sparse checkout it does a full checkout. + * `:depth` - (since v1.17.0) creates a shallow clone of the Git repository, + limiting the history to the specified number of commits. This can significantly + improve clone speed for large repositories when full history is not needed. + The value must be a positive integer, typically `1`. If your Git repository requires authentication, such as basic username:password HTTP authentication via URLs, it can be achieved via Git configuration, keeping diff --git a/lib/mix/test/mix/scm/git_test.exs b/lib/mix/test/mix/scm/git_test.exs index 24429510544..d572ba06f0c 100644 --- a/lib/mix/test/mix/scm/git_test.exs +++ b/lib/mix/test/mix/scm/git_test.exs @@ -32,7 +32,7 @@ defmodule Mix.SCM.GitTest do "https://github.com/elixir-lang/some_dep.git - v1" assert Mix.SCM.Git.format(Keyword.put(opts, :branch, "b")) == - "https://github.com/elixir-lang/some_dep.git - origin/b" + "https://github.com/elixir-lang/some_dep.git - b" assert Mix.SCM.Git.format(Keyword.put(opts, :ref, "abcdef")) == "https://github.com/elixir-lang/some_dep.git - abcdef" diff --git a/lib/mix/test/mix/tasks/deps.git_test.exs b/lib/mix/test/mix/tasks/deps.git_test.exs index 8983a6abb73..a5338b2db3e 100644 --- a/lib/mix/test/mix/tasks/deps.git_test.exs +++ b/lib/mix/test/mix/tasks/deps.git_test.exs @@ -478,6 +478,207 @@ defmodule Mix.Tasks.DepsGitTest do purge([GitRepo, GitRepo.MixProject]) end + describe "Git depth option" do + @describetag :git_depth + + test "gets and updates Git repos with depth option" do + Process.put(:git_repo_opts, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + # Expand depth + update_dep(depth: 2) + Mix.Tasks.Deps.Get.run([]) + assert_shallow("deps/git_repo", 2) + + # Reduce depth + update_dep(depth: 1) + Mix.Tasks.Deps.Get.run([]) + assert_shallow("deps/git_repo", 1) + end) + end + + test "with tag" do + Process.put(:git_repo_opts, depth: 1, tag: "with_module") + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - with_module)" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + end) + end + + test "with branch" do + Process.put(:git_repo_opts, depth: 1, branch: "main") + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - main)" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + end) + end + + test "with ref" do + [last, _ | _] = get_git_repo_revs("git_repo") + + Process.put(:git_repo_opts, depth: 1, ref: last) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - #{last})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + end) + end + + test "changing refspec updates retaining depth" do + [last, first | _] = get_git_repo_revs("git_repo") + + Process.put(:git_repo_opts, ref: first, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")} - #{first})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + assert File.read!("mix.lock") =~ first + + # Change refspec + update_dep(ref: last, depth: 1) + Mix.Tasks.Deps.Get.run([]) + assert_shallow("deps/git_repo", 1) + assert File.read!("mix.lock") =~ last + end) + end + + test "removing depth retains shallow repository" do + # For compatibility and simplicity, we follow Git's behavior and do not + # attempt to unshallow an existing repository. This should not be a + # problem, because all we guarantee is that the correct source code is + # available whenever mix.exs or mix.lock change. If one wanted to have a + # full clone, they can always run `deps.clean` and `deps.get` again. + Process.put(:git_repo_opts, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + # Remove depth + update_dep([]) + Mix.Tasks.Deps.Get.run([]) + refute File.read!("mix.lock") =~ "depth:" + assert File.exists?("deps/git_repo/.git/shallow") + + assert System.cmd("git", ~w[--git-dir=deps/git_repo/.git rev-list --count HEAD]) == + {"1\n", 0} + end) + end + + @tag :git_sparse + test "with sparse checkout" do + Process.put(:git_repo_opts, sparse: "sparse_dir", depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + refute File.exists?("deps/git_repo/mix.exs") + assert File.exists?("deps/git_repo/sparse_dir/mix.exs") + assert File.read!("mix.lock") =~ "sparse: \"sparse_dir\"" + end) + end + + test "with subdir" do + Process.put(:git_repo_opts, subdir: "sparse_dir", depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + assert File.exists?("deps/git_repo/mix.exs") + assert File.exists?("deps/git_repo/sparse_dir/mix.exs") + assert File.read!("mix.lock") =~ "subdir: \"sparse_dir\"" + end) + end + + test "does not affect submodules depth" do + # The expectation is that we can add an explicit option in the future, + # just like git-clone has `--shallow-submodules`. + Process.put(:git_repo_opts, submodules: true, depth: 1) + + in_fixture("no_mixfile", fn -> + Mix.Project.push(GitApp) + + Mix.Tasks.Deps.Get.run([]) + message = "* Getting git_repo (#{fixture_path("git_repo")})" + assert_received {:mix_shell, :info, [^message]} + assert_shallow("deps/git_repo", 1) + + assert File.read!("mix.lock") =~ "submodules: true" + # TODO: assert submodule is not shallow. This would likely require + # changes to the fixtures. Apparently, not even the submodules-specific + # tests check that the cloned repo contains submodules as expected. + end) + end + + defp update_dep(git_repo_opts) do + # Flush the errors we got, move to a clean slate + Mix.shell().flush() + Mix.Task.clear() + Process.put(:git_repo_opts, git_repo_opts) + Mix.Project.pop() + Mix.Project.push(GitApp) + end + + defp assert_shallow(repo_path, depth) do + assert File.read!("mix.lock") =~ "depth: #{depth}" + + # Check if the repository is a shallow clone + assert File.exists?(repo_path <> "/.git/shallow") + + # Check the number of commits in the current branch. + # + # We could consider all branches with `git rev-list --count --all`, as in + # practice there should be only a single branch. However, the test fixture + # sets up two branches, and that brings us to an interesting situation: + # instead of guaranteeing that the `:depth` option would keep the + # repository lean even after refspec changes, we only guarantee the number + # of commits in the current branch, perhaps leaving more objects around + # than strictly necessary. This allows us to keep the implementation + # simple, while still providing a reasonable guarantee. + assert System.cmd("git", ~w[--git-dir=#{repo_path}/.git rev-list --count HEAD]) == + {"#{depth}\n", 0} + end + end + defp refresh(post_config) do %{name: name, file: file} = Mix.Project.pop() Mix.ProjectStack.post_config(post_config) diff --git a/lib/mix/test/test_helper.exs b/lib/mix/test/test_helper.exs index 61733a02c5f..c88194a04cb 100644 --- a/lib/mix/test/test_helper.exs +++ b/lib/mix/test/test_helper.exs @@ -22,7 +22,13 @@ Application.put_env(:logger, :backends, []) os_exclude = if match?({:win32, _}, :os.type()), do: [unix: true], else: [windows: true] epmd_exclude = if match?({:win32, _}, :os.type()), do: [epmd: true], else: [] -git_exclude = if Mix.SCM.Git.git_version() <= {1, 7, 4}, do: [git_sparse: true], else: [] + +git_exclude = + Mix.SCM.Git.unsupported_options() + |> Enum.map(fn + :sparse -> {:git_sparse, true} + :depth -> {:git_depth, true} + end) {line_exclude, line_include} = if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}