Skip to content

Optimistically shallow clone submodules #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
joshheinrichs-shopify wants to merge 1 commit into
base: tecnix-gcs
Choose a base branch
from
Draft

Optimistically shallow clone submodules #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 43 additions & 20 deletions src/libfetchers/git.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1180,27 +1180,50 @@ struct GitInputScheme : InputScheme
submodule.branch,
submoduleRev.gitRev(),
resolved);
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", resolved);
if (submodule.branch != "") {
// A special value of . is used to indicate that the name of the branch in the submodule
// should be the same name as the current branch in the current repository.
// https://git-scm.com/docs/gitmodules
if (submodule.branch == ".") {
attrs.insert_or_assign("ref", ref);
} else {
attrs.insert_or_assign("ref", submodule.branch);
auto fetchSubmodule = [&](bool useShallow) {
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", resolved);
if (submodule.branch != "") {
// A special value of . is used to indicate that the name of the branch in the submodule
// should be the same name as the current branch in the current repository.
// https://git-scm.com/docs/gitmodules
if (submodule.branch == ".") {
attrs.insert_or_assign("ref", ref);
} else {
attrs.insert_or_assign("ref", submodule.branch);
}
}
}
attrs.insert_or_assign("rev", submoduleRev.gitRev());
attrs.insert_or_assign("exportIgnore", Explicit<bool>{options.exportIgnore});
attrs.insert_or_assign("submodules", Explicit<bool>{true});
attrs.insert_or_assign("lfs", Explicit<bool>{options.smudgeLfs});
attrs.insert_or_assign("allRefs", Explicit<bool>{true});
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»");
attrs.insert_or_assign("rev", submoduleRev.gitRev());
attrs.insert_or_assign("exportIgnore", Explicit<bool>{options.exportIgnore});
attrs.insert_or_assign("submodules", Explicit<bool>{true});
attrs.insert_or_assign("lfs", Explicit<bool>{options.smudgeLfs});
if (useShallow)
attrs.insert_or_assign("shallow", Explicit<bool>{true});
else
attrs.insert_or_assign("allRefs", Explicit<bool>{true});

auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»");
return submoduleAccessor;
};

auto submoduleAccessor = [&]() {
if (shallow) {
try {
return fetchSubmodule(true);
} catch (Error & e) {
debug(
"shallow fetch of Git submodule '%s' from '%s' failed: %s; falling back to full fetch",
submodule.path,
resolved,
e.what());
}
}
return fetchSubmodule(false);
}();

mounts.insert_or_assign(submodule.path, submoduleAccessor);
}

Expand Down
96 changes: 96 additions & 0 deletions tests/functional/fetchGitSubmodules.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,102 @@ pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url =
[[ -e $pathWithoutExportIgnore/exclude-from-root ]]
[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]

test_submodule_shallow_fast_path() {
local repoA=$TEST_ROOT/submodule_shallow/a
local repoB=$TEST_ROOT/submodule_shallow/b

rm -rf "$TEST_HOME"/.cache/nix

createGitRepo "$repoB"
addGitContent "$repoB"
local firstRev
firstRev=$(git -C "$repoB" rev-parse HEAD)
echo "dolor sit amet" > "$repoB"/content
git -C "$repoB" commit -am "Second commit"
local subRev
subRev=$(git -C "$repoB" rev-parse HEAD)

createGitRepo "$repoA"
git -C "$repoA" submodule add "$repoB" b
git -C "$repoA" add b
addGitContent "$repoA"

local rev
rev=$(git -C "$repoA" rev-parse HEAD)
local out
out=$(_NIX_FORCE_HTTP=1 nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; shallow = true; submodules = true; }).outPath")
test -e "$out"/b/content

local submoduleCacheRepos=()
for cacheRepo in "$TEST_HOME"/.cache/nix/gitv3/*; do
[[ -d "$cacheRepo"/objects ]] || continue
if git -C "$cacheRepo" --git-dir . cat-file -e "$subRev^{commit}" 2>/dev/null; then
submoduleCacheRepos+=("$cacheRepo")
[[ $(git -C "$cacheRepo" --git-dir . rev-parse --is-shallow-repository) == true ]]
if git -C "$cacheRepo" --git-dir . cat-file -e "$firstRev^{commit}" 2>/dev/null; then
fail "submodule cache unexpectedly contains previous commit"
fi
fi
done
[[ ${#submoduleCacheRepos[@]} == 1 ]]
}
test_submodule_shallow_fast_path

test_submodule_shallow_fallback() {
local repoA=$TEST_ROOT/submodule_shallow_fallback/a
local repoB=$TEST_ROOT/submodule_shallow_fallback/b

rm -rf "$TEST_HOME"/.cache/nix

createGitRepo "$repoB"
echo "one" > "$repoB"/content
git -C "$repoB" add content
git -C "$repoB" commit -m "First commit"
echo "two" > "$repoB"/content
git -C "$repoB" commit -am "Second commit"
local pinnedRev
pinnedRev=$(git -C "$repoB" rev-parse HEAD)
echo "three" > "$repoB"/content
git -C "$repoB" commit -am "Third commit"
local tipRev
tipRev=$(git -C "$repoB" rev-parse HEAD)

createGitRepo "$repoA"
git -C "$repoA" submodule add "$repoB" b
git -C "$repoA"/b checkout --quiet "$pinnedRev"
echo "root" > "$repoA"/content
git -C "$repoA" add .gitmodules b content
git -C "$repoA" commit -m "Add submodule"

local rev
rev=$(git -C "$repoA" rev-parse HEAD)
local out
# Protocol v1 rejects shallow fetching the pinned non-tip commit by SHA,
# but the full all-refs fallback can fetch it.
out=$(
GIT_CONFIG_COUNT=2 \
GIT_CONFIG_KEY_0=protocol.file.allow \
GIT_CONFIG_VALUE_0=always \
GIT_CONFIG_KEY_1=protocol.version \
GIT_CONFIG_VALUE_1=1 \
_NIX_FORCE_HTTP=1 \
nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; shallow = true; submodules = true; }).outPath"
)
[[ $(< "$out"/b/content) == two ]]

local submoduleCacheRepos=()
for cacheRepo in "$TEST_HOME"/.cache/nix/gitv3/*; do
[[ -d "$cacheRepo"/objects ]] || continue
if git -C "$cacheRepo" --git-dir . cat-file -e "$pinnedRev^{commit}" 2>/dev/null; then
submoduleCacheRepos+=("$cacheRepo")
[[ $(git -C "$cacheRepo" --git-dir . rev-parse --is-shallow-repository) == false ]]
git -C "$cacheRepo" --git-dir . cat-file -e "$tipRev^{commit}"
fi
done
[[ ${#submoduleCacheRepos[@]} == 1 ]]
}
test_submodule_shallow_fallback

test_submodule_nested() {
local repoA=$TEST_ROOT/submodule_nested/a
local repoB=$TEST_ROOT/submodule_nested/b
Expand Down
Loading