From 3531a04c6ffa394947204cee2cf189aa708a3ba2 Mon Sep 17 00:00:00 2001 From: Josh Heinrichs Date: Tue, 23 Jun 2026 23:03:50 -0600 Subject: [PATCH] Optimistically shallow clone submodules In chonky repos like grpc, full clones of submodules can take on the order of ~3 minutes. Shallow cloning should help though I haven't benchmarked this yet. In order to avoid changing observable behaviour, we fall back to a regular clone so if someone was shallow cloning a repo with submodules, and the submodule was coming from a forge that doesn't allow cloning of a specific sha, the existing cloning behaviour should be preserved. --- src/libfetchers/git.cc | 63 +++++++++++------ tests/functional/fetchGitSubmodules.sh | 96 ++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5273213a9..b9ca08d71 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1180,27 +1180,50 @@ struct GitInputScheme : InputScheme submodule.branch, submoduleRev.gitRev(), resolved); - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", resolved); - if (submodule.branch != "") { - // A special value of . is used to indicate that the name of the branch in the submodule - // should be the same name as the current branch in the current repository. - // https://git-scm.com/docs/gitmodules - if (submodule.branch == ".") { - attrs.insert_or_assign("ref", ref); - } else { - attrs.insert_or_assign("ref", submodule.branch); + auto fetchSubmodule = [&](bool useShallow) { + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") { + // A special value of . is used to indicate that the name of the branch in the submodule + // should be the same name as the current branch in the current repository. + // https://git-scm.com/docs/gitmodules + if (submodule.branch == ".") { + attrs.insert_or_assign("ref", ref); + } else { + attrs.insert_or_assign("ref", submodule.branch); + } } - } - attrs.insert_or_assign("rev", submoduleRev.gitRev()); - attrs.insert_or_assign("exportIgnore", Explicit{options.exportIgnore}); - attrs.insert_or_assign("submodules", Explicit{true}); - attrs.insert_or_assign("lfs", Explicit{options.smudgeLfs}); - attrs.insert_or_assign("allRefs", Explicit{true}); - auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); - auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); - submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{options.exportIgnore}); + attrs.insert_or_assign("submodules", Explicit{true}); + attrs.insert_or_assign("lfs", Explicit{options.smudgeLfs}); + if (useShallow) + attrs.insert_or_assign("shallow", Explicit{true}); + else + attrs.insert_or_assign("allRefs", Explicit{true}); + + auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); + return submoduleAccessor; + }; + + auto submoduleAccessor = [&]() { + if (shallow) { + try { + return fetchSubmodule(true); + } catch (Error & e) { + debug( + "shallow fetch of Git submodule '%s' from '%s' failed: %s; falling back to full fetch", + submodule.path, + resolved, + e.what()); + } + } + return fetchSubmodule(false); + }(); + mounts.insert_or_assign(submodule.path, submoduleAccessor); } diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index 7839fef0d..3ba547970 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -190,6 +190,102 @@ pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = [[ -e $pathWithoutExportIgnore/exclude-from-root ]] [[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] +test_submodule_shallow_fast_path() { + local repoA=$TEST_ROOT/submodule_shallow/a + local repoB=$TEST_ROOT/submodule_shallow/b + + rm -rf "$TEST_HOME"/.cache/nix + + createGitRepo "$repoB" + addGitContent "$repoB" + local firstRev + firstRev=$(git -C "$repoB" rev-parse HEAD) + echo "dolor sit amet" > "$repoB"/content + git -C "$repoB" commit -am "Second commit" + local subRev + subRev=$(git -C "$repoB" rev-parse HEAD) + + createGitRepo "$repoA" + git -C "$repoA" submodule add "$repoB" b + git -C "$repoA" add b + addGitContent "$repoA" + + local rev + rev=$(git -C "$repoA" rev-parse HEAD) + local out + out=$(_NIX_FORCE_HTTP=1 nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; shallow = true; submodules = true; }).outPath") + test -e "$out"/b/content + + local submoduleCacheRepos=() + for cacheRepo in "$TEST_HOME"/.cache/nix/gitv3/*; do + [[ -d "$cacheRepo"/objects ]] || continue + if git -C "$cacheRepo" --git-dir . cat-file -e "$subRev^{commit}" 2>/dev/null; then + submoduleCacheRepos+=("$cacheRepo") + [[ $(git -C "$cacheRepo" --git-dir . rev-parse --is-shallow-repository) == true ]] + if git -C "$cacheRepo" --git-dir . cat-file -e "$firstRev^{commit}" 2>/dev/null; then + fail "submodule cache unexpectedly contains previous commit" + fi + fi + done + [[ ${#submoduleCacheRepos[@]} == 1 ]] +} +test_submodule_shallow_fast_path + +test_submodule_shallow_fallback() { + local repoA=$TEST_ROOT/submodule_shallow_fallback/a + local repoB=$TEST_ROOT/submodule_shallow_fallback/b + + rm -rf "$TEST_HOME"/.cache/nix + + createGitRepo "$repoB" + echo "one" > "$repoB"/content + git -C "$repoB" add content + git -C "$repoB" commit -m "First commit" + echo "two" > "$repoB"/content + git -C "$repoB" commit -am "Second commit" + local pinnedRev + pinnedRev=$(git -C "$repoB" rev-parse HEAD) + echo "three" > "$repoB"/content + git -C "$repoB" commit -am "Third commit" + local tipRev + tipRev=$(git -C "$repoB" rev-parse HEAD) + + createGitRepo "$repoA" + git -C "$repoA" submodule add "$repoB" b + git -C "$repoA"/b checkout --quiet "$pinnedRev" + echo "root" > "$repoA"/content + git -C "$repoA" add .gitmodules b content + git -C "$repoA" commit -m "Add submodule" + + local rev + rev=$(git -C "$repoA" rev-parse HEAD) + local out + # Protocol v1 rejects shallow fetching the pinned non-tip commit by SHA, + # but the full all-refs fallback can fetch it. + out=$( + GIT_CONFIG_COUNT=2 \ + GIT_CONFIG_KEY_0=protocol.file.allow \ + GIT_CONFIG_VALUE_0=always \ + GIT_CONFIG_KEY_1=protocol.version \ + GIT_CONFIG_VALUE_1=1 \ + _NIX_FORCE_HTTP=1 \ + nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; shallow = true; submodules = true; }).outPath" + ) + [[ $(< "$out"/b/content) == two ]] + + local submoduleCacheRepos=() + for cacheRepo in "$TEST_HOME"/.cache/nix/gitv3/*; do + [[ -d "$cacheRepo"/objects ]] || continue + if git -C "$cacheRepo" --git-dir . cat-file -e "$pinnedRev^{commit}" 2>/dev/null; then + submoduleCacheRepos+=("$cacheRepo") + [[ $(git -C "$cacheRepo" --git-dir . rev-parse --is-shallow-repository) == false ]] + git -C "$cacheRepo" --git-dir . cat-file -e "$tipRev^{commit}" + fi + done + [[ ${#submoduleCacheRepos[@]} == 1 ]] +} +test_submodule_shallow_fallback + test_submodule_nested() { local repoA=$TEST_ROOT/submodule_nested/a local repoB=$TEST_ROOT/submodule_nested/b