From 2a74a94a9c1296f0c9e774d167838b246828b734 Mon Sep 17 00:00:00 2001 From: montehurd Date: Thu, 15 Oct 2026 04:07:30 -0700 Subject: [PATCH 1/9] Serve Git repositories from a local cache Named upstream.git hosts are served read-only over Git's smart HTTP protocol at /git/{name}/{repo}, so git clone and git fetch work against the proxy through an insteadOf rewrite. Each repository is a bare clone at git.cache_dir/{name}-{hash of the upstream URL}/{repo}.git, served by git upload-pack. Every request for refs checks the upstream's branches, tags and HEAD with a protocol v2 ls-refs limited to those prefixes, and fetches only when they changed, atomically and with --prune. Concurrent requests share one refresh. Only branches and tags are cached, and HEAD follows the upstream's default branch. A commit a client asks for by ID that the cache lacks, such as a pull request head, is fetched and kept under a hidden ref until a branch or tag includes it or for 7 days. One the upstream cannot provide is not asked for again for a minute, or an hour when it is larger than git.max_repo_size leaves. This is off for upstreams upstream.auth has credentials for, since GitHub serves commits from any repository in a fork network. git never reaches an upstream itself. Its fetches go through a loopback relay that forwards with the proxy's upstream client, so address checks, upstream.auth and the access log apply, and git never sees a credential. Redirects are not followed, and git runs with an environment of its own. The cached copy is served while the upstream fails or is slower than git.refresh_wait, until git.stale_max_age has passed since the last successful check. Refusals and missing repositories are never answered from the cache. git.max_repo_size cuts a transfer off as it streams, and every fetched pack is kept as a pack, so a cut-off leaves nothing behind. A repository that can no longer be refreshed within the limit is cloned again with only its branches and tags. git.max_size evicts the least recently used repositories, never one in use. At startup only the directory of a repointed upstream's old URL is removed, and Close waits for refreshes in progress. Hosts with upstream.auth credentials need a repos allowlist, and a repository outside it answers 404. Refs #403 --- Dockerfile | 2 +- README.md | 22 + config.example.yaml | 17 + docs/architecture.md | 15 + docs/configuration.md | 92 ++++ internal/config/config.go | 20 + internal/config/git.go | 222 ++++++++++ internal/config/git_test.go | 172 ++++++++ internal/gitcache/cache.go | 421 ++++++++++++++++++ internal/gitcache/cache_test.go | 659 +++++++++++++++++++++++++++++ internal/gitcache/git.go | 100 +++++ internal/gitcache/helpers_test.go | 349 +++++++++++++++ internal/gitcache/pktline.go | 65 +++ internal/gitcache/refresh.go | 336 +++++++++++++++ internal/gitcache/relay.go | 175 ++++++++ internal/gitcache/relay_test.go | 96 +++++ internal/gitcache/serve.go | 151 +++++++ internal/gitcache/serve_test.go | 196 +++++++++ internal/gitcache/upstream.go | 210 +++++++++ internal/gitcache/wants.go | 264 ++++++++++++ internal/gitcache/wants_test.go | 319 ++++++++++++++ internal/handler/git.go | 213 ++++++++++ internal/handler/git_test.go | 247 +++++++++++ internal/server/git_test.go | 160 +++++++ internal/server/middleware.go | 2 +- internal/server/middleware_test.go | 2 +- internal/server/server.go | 33 ++ 27 files changed, 4557 insertions(+), 3 deletions(-) create mode 100644 internal/config/git.go create mode 100644 internal/config/git_test.go create mode 100644 internal/gitcache/cache.go create mode 100644 internal/gitcache/cache_test.go create mode 100644 internal/gitcache/git.go create mode 100644 internal/gitcache/helpers_test.go create mode 100644 internal/gitcache/pktline.go create mode 100644 internal/gitcache/refresh.go create mode 100644 internal/gitcache/relay.go create mode 100644 internal/gitcache/relay_test.go create mode 100644 internal/gitcache/serve.go create mode 100644 internal/gitcache/serve_test.go create mode 100644 internal/gitcache/upstream.go create mode 100644 internal/gitcache/wants.go create mode 100644 internal/gitcache/wants_test.go create mode 100644 internal/handler/git.go create mode 100644 internal/handler/git_test.go create mode 100644 internal/server/git_test.go diff --git a/Dockerfile b/Dockerfile index de784e82..41139993 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o / FROM alpine:3.24.2 -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates git COPY --from=builder /proxy /usr/local/bin/proxy diff --git a/README.md b/README.md index 6ab21cb1..1ca1f0db 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Arch | Arch Linux | | ✓ | | Chef Supermarket | Chef | | ✓ | | Generic | Any | | ✓ | +| Git | Any | | ✓ | | Helm | Kubernetes | Yes | ✓ | | Vagrant | Vagrant | | ✗ | @@ -824,6 +825,26 @@ go.download_mirror = "http://localhost:8080/generic/golang" Other download hosts need their version-pinned paths declared in `upstream.generic_artifacts`. +### Git repositories + +Configure named Git servers: + +```yaml +upstream: + git: + github: "https://github.com" +``` + +Then point clones at the proxy: + +```bash +git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ +``` + +Each repository is cached as a bare clone and refreshed whenever its branches +or tags change upstream. See [configuration](docs/configuration.md#git-repositories) +for size limits, private repositories and outage behavior. + ## Configuration The proxy can be configured via: @@ -1148,6 +1169,7 @@ Recently cached: | `GET /apk/{repository}/*` | Alpine APK repository protocol | | `GET /arch/{repository}/*` | Arch Linux pacman repository protocol | | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | +| `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only | | `GET /debian/*` | Debian/APT repository protocol (main archive) | | `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) | | `GET /rpm/*` | RPM/Yum repository protocol | diff --git a/config.example.yaml b/config.example.yaml index 277eb231..1a0ea2f2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -326,6 +326,15 @@ upstream: # apache: # - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + # Named Git servers (used by /git/{name}/{repo}), served read-only from + # cached bare repositories. repos limits which repositories may be served + # and is required when upstream.auth has credentials for the server. + # git: + # github: "https://github.com" + # wikimedia: + # url: "https://gerrit.wikimedia.org/r" + # repos: ["mediawiki/**"] + # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path # segment boundaries must match; the longest matching scope wins. @@ -366,6 +375,14 @@ upstream: # "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com": # type: ecr +# Git repository cache, used when upstream.git is set +# git: +# cache_dir: ./cache/git # local disk, used by this process only +# max_size: "" # evict least recently used repositories past this +# max_repo_size: 5GB # refuse a repository that grows past this +# refresh_wait: 5s # then serve the cached copy if the upstream is slow +# stale_max_age: 24h # never serve a copy not checked for this long + # Gradle HttpBuildCache configuration gradle: build_cache: diff --git a/docs/architecture.md b/docs/architecture.md index a259e910..f9325b25 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -301,6 +301,21 @@ HTTP protocol handlers for each registry type. - Proxies the Swift Package Registry v1 read endpoints - Rewrites release URLs and caches source archives +**GitHandler:** +- Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP +- Validates repository paths and the `repos` allowlist, and maps cache errors to statuses + +### `internal/gitcache` + +Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed. + +- Each request for refs checks the upstream's branches and tags with protocol v2 `ls-refs` and fetches only when they changed; concurrent requests share one refresh +- Clients are served by `git upload-pack --stateless-rpc` +- Spawned `git` reaches the upstream only through a loopback relay that forwards with the proxy's upstream client, so address checks, `upstream.auth` and the access log apply +- Commits a client asks for by ID that are on no cached branch or tag are fetched and kept under hidden `refs/proxy/wanted/{time}/{id}` refs until a branch or tag includes them or for 7 days; off for upstreams with credentials +- Evicts least recently used repositories after each fetch once `git.max_size` is passed, and drops a repository that can no longer be refreshed within `git.max_repo_size` so it is cloned again; never one in use. A commit fetched by ID that does not fit is refused instead +- Every fetched pack is kept as a pack (`transfer.unpackLimit=1`), so a transfer cut off at the size limit leaves nothing behind + ### `internal/server` HTTP server setup, web UI, and API handlers. diff --git a/docs/configuration.md b/docs/configuration.md index dccdf265..f755ba4c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -623,6 +623,98 @@ those names unless the shadowing is intended. This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. +### Git repositories + +Git servers configured under `upstream.git` are served read-only at +`/git/{name}/{repo}` over Git's smart HTTP protocol, from bare repositories +cached on local disk: + +```yaml +upstream: + git: + github: "https://github.com" + wikimedia: + url: "https://gerrit.wikimedia.org/r" + repos: ["mediawiki/core", "mediawiki/extensions/**"] # optional allowlist + +git: + cache_dir: /data/git # local disk, used by this process only + max_size: 50GB # evict least recently used repositories past this + max_repo_size: 5GB # refuse a repository that grows past this + refresh_wait: 5s # then serve the cached copy if the upstream is slow + stale_max_age: 24h # never serve a copy not checked for this long +``` + +Clients point their remote URLs at the proxy, for example with `insteadOf`: + +```bash +git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ +``` + +`repos` patterns use `path.Match` syntax, where `*` does not cross a `/`; a +pattern ending in `/**` also matches every repository nested below its +prefix, so `mediawiki/**` matches `mediawiki/skins/Vector`. An upstream +without `repos` serves every repository on it, and a repository outside +`repos` answers 404. When `upstream.auth` has credentials for an upstream, +`repos` is required: everyone who can reach the proxy can read every +repository it serves, so list only those they may read. Per-client +permissions are not supported. + +Every request for a repository's refs checks the upstream's branches and tags, +and fetches only when they changed, so a clone right after a push sees the new +commit. Requests that arrive while a check or fetch is running share it. Only +branches and tags are cached and served; other refs, such as Gerrit's +`refs/changes/*`, are left out, and protocol v2 lets the server filter them +before sending. + +A commit a client asks for by object ID that is on no cached branch or tag, +such as a pull request head fetched by `actions/checkout`, is fetched from the +upstream when the client asks for it, and kept until a cached branch or tag +includes it or for 7 days. A commit the upstream could not provide, or could +not be reached for, is not asked for again for a minute, or for an hour when +it is larger than what `max_repo_size` leaves. This is off for upstreams that `upstream.auth` has +credentials for: GitHub serves a commit from any repository in a fork network +through any other, so a fetch by ID could reach repositories outside `repos`. +Any commit the cache holds can be fetched by ID, including one a force-push +left behind, as from the upstream. + +If the upstream fails or takes longer than `refresh_wait`, a repository checked +successfully within `stale_max_age` is served from the cache while the refresh +continues. A `stale_max_age` of `0` never serves a copy the upstream has not +just confirmed. A repository the upstream reports missing (404), refuses (401, 403) +or redirects is not served from the cache. Redirects are never followed, so +point the upstream at a repository's current location. + +The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). +The `git` processes never reach the network themselves: their fetches go +through a loopback endpoint in the proxy that forwards them with the same +upstream client as every other request, so the address checks +(`allow_loopback`, `allow_private_hosts`), `upstream.auth` and the access log +apply, and credentials are never passed to `git`. + +`max_repo_size` defaults to 5GB, and `max_size` to no limit. A clone or fetch +that would pass `max_repo_size` is stopped, and nothing of it is kept. A new +repository is then refused for an hour. When a refresh of one already cached +does not fit, it is served as it is and dropped once no request is reading +it, so the next request clones it again with only its branches and tags; if +that is still too large, it is refused for an hour. A commit fetched by ID +that does not fit is refused, and the repository kept. +Least recently used repositories are evicted after each fetch that takes the +cache past `max_size`, never one a request is reading. A transfer +that receives nothing for 10 minutes is abandoned. Git's automatic garbage +collection runs as part of fetches. + +Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so +repointing an upstream starts a fresh cache, and the old URL's directory is +removed at startup. Nothing else in `cache_dir` is touched, and the +directories of upstreams no longer configured are only logged, so remove +those yourself. + +Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings +under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`, +`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and +`PROXY_GIT_STALE_MAX_AGE`. + ## Authentication Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. diff --git a/internal/config/config.go b/internal/config/config.go index e9976deb..6f5a3522 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -216,6 +216,9 @@ type Config struct { // Health configures the /health endpoint behavior. Health HealthConfig `json:"health" yaml:"health"` + // Git configures the Git repository cache served at /git/{name}/. + Git GitConfig `json:"git" yaml:"git"` + // envWarnings collects environment values LoadFromEnv ignored. envWarnings []string } @@ -767,6 +770,11 @@ type UpstreamConfig struct { // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // Git maps names to Git servers, served read-only at /git/{name}/{repo} + // from a local cache of bare repositories. See GitUpstream. + // Example: {"github": {"url": "https://github.com"}}. + Git map[string]GitUpstream `json:"git" yaml:"git"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -837,6 +845,9 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := u.validateGitUpstreams(); err != nil { + return err + } if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { return err } @@ -1281,6 +1292,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") + setEnvString(&c.Git.CacheDir, "PROXY_GIT_CACHE_DIR") + setEnvString(&c.Git.MaxSize, "PROXY_GIT_MAX_SIZE") + setEnvString(&c.Git.MaxRepoSize, "PROXY_GIT_MAX_REPO_SIZE") + setEnvString(&c.Git.RefreshWait, "PROXY_GIT_REFRESH_WAIT") + setEnvString(&c.Git.StaleMaxAge, "PROXY_GIT_STALE_MAX_AGE") } // validateAbsoluteURL returns an error if value is not a parseable URL with @@ -1437,6 +1453,10 @@ func (c *Config) validateComponents() error { return err } + if err := c.Git.Validate(); err != nil { + return err + } + return c.Gradle.BuildCache.Validate() } diff --git a/internal/config/git.go b/internal/config/git.go new file mode 100644 index 00000000..8a0028d2 --- /dev/null +++ b/internal/config/git.go @@ -0,0 +1,222 @@ +package config + +import ( + "encoding/json" + "fmt" + "path" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + defaultGitCacheDir = "./cache/git" + defaultGitMaxRepoSize = "5GB" + defaultGitRefreshWait = "5s" + defaultGitStaleMaxAge = "24h" +) + +// GitUpstream is an upstream.git entry: a Git server's base URL and, +// optionally, the repositories under it that may be served. In YAML or JSON +// it is either the URL alone or an object with url and repos. +type GitUpstream struct { + URL string `json:"url" yaml:"url"` + + // Repos lists path.Match patterns for repository paths under URL, such + // as "mediawiki/*". A pattern ending in "/**" also matches every + // repository nested below its prefix: "mediawiki/**" matches + // mediawiki/skins/Vector. Empty allows every repository. Required when + // upstream.auth has credentials for the URL, since everyone who can + // reach the proxy can read what it serves. + Repos []string `json:"repos" yaml:"repos"` +} + +// UnmarshalYAML accepts a bare URL as well as the object form. +func (g *GitUpstream) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return node.Decode(&g.URL) + } + type plain GitUpstream + return node.Decode((*plain)(g)) +} + +// UnmarshalJSON accepts a bare URL as well as the object form. +func (g *GitUpstream) UnmarshalJSON(data []byte) error { + if err := json.Unmarshal(data, &g.URL); err == nil { + return nil + } + type plain GitUpstream + return json.Unmarshal(data, (*plain)(g)) +} + +// AllowsRepo reports whether repo, a path under URL, may be served. +func (g GitUpstream) AllowsRepo(repo string) bool { + if len(g.Repos) == 0 { + return true + } + for _, pattern := range g.Repos { + if matchRepoPattern(pattern, repo) { + return true + } + } + return false +} + +func matchRepoPattern(pattern, repo string) bool { + prefix, nested := strings.CutSuffix(pattern, "/**") + if !nested { + ok, _ := path.Match(pattern, repo) + return ok + } + segments := strings.Count(prefix, "/") + 1 + parts := strings.SplitN(repo, "/", segments+1) + if len(parts) <= segments { + return false + } + ok, _ := path.Match(prefix, strings.Join(parts[:segments], "/")) + return ok +} + +// GitConfig configures the Git repository cache behind /git/{name}/. +type GitConfig struct { + // CacheDir holds the cached bare repositories. It must be local disk + // used by this process only. Default: ./cache/git. + CacheDir string `json:"cache_dir" yaml:"cache_dir"` + + // MaxSize evicts the least recently used repositories once their total + // size passes it. Empty or "0" disables the limit. + MaxSize string `json:"max_size" yaml:"max_size"` + + // MaxRepoSize is the most one repository may take; a clone or fetch that + // would pass it fails. "0" disables the limit. Default: 5GB. + MaxRepoSize string `json:"max_repo_size" yaml:"max_repo_size"` + + // RefreshWait is how long a request waits for a repository to be + // refreshed from upstream before it is served the cached copy. + // Default: 5s. + RefreshWait string `json:"refresh_wait" yaml:"refresh_wait"` + + // StaleMaxAge is how long after the last successful upstream check a + // cached copy may still be served when the upstream fails or is slow. + // Default: 24h. + StaleMaxAge string `json:"stale_max_age" yaml:"stale_max_age"` + + maxSize int64 + maxRepoSize int64 + refreshWait time.Duration + staleMaxAge time.Duration +} + +// MaxSizeBytes returns the parsed MaxSize, 0 for no limit. Validate must run first. +func (g *GitConfig) MaxSizeBytes() int64 { return g.maxSize } + +// MaxRepoSizeBytes returns the parsed MaxRepoSize, 0 for no limit. Validate must run first. +func (g *GitConfig) MaxRepoSizeBytes() int64 { return g.maxRepoSize } + +// RefreshWaitDuration returns the parsed RefreshWait. Validate must run first. +func (g *GitConfig) RefreshWaitDuration() time.Duration { return g.refreshWait } + +// StaleMaxAgeDuration returns the parsed StaleMaxAge. Validate must run first. +func (g *GitConfig) StaleMaxAgeDuration() time.Duration { return g.staleMaxAge } + +// Validate checks the Git cache settings, applying defaults where unset. +func (g *GitConfig) Validate() error { + if g.CacheDir == "" { + g.CacheDir = defaultGitCacheDir + } + if g.MaxRepoSize == "" { + g.MaxRepoSize = defaultGitMaxRepoSize + } + if g.RefreshWait == "" { + g.RefreshWait = defaultGitRefreshWait + } + if g.StaleMaxAge == "" { + g.StaleMaxAge = defaultGitStaleMaxAge + } + + var err error + if g.maxSize, err = parseNonNegativeSize("git.max_size", g.MaxSize); err != nil { + return err + } + if g.maxRepoSize, err = parseNonNegativeSize("git.max_repo_size", g.MaxRepoSize); err != nil { + return err + } + if g.refreshWait, err = parseNonNegativeDuration("git.refresh_wait", g.RefreshWait); err != nil { + return err + } + if g.staleMaxAge, err = parseNonNegativeDuration("git.stale_max_age", g.StaleMaxAge); err != nil { + return err + } + return nil +} + +func parseNonNegativeSize(field, value string) (int64, error) { + size, err := ParseSize(value) + if err != nil { + return 0, fmt.Errorf("invalid %s: %w", field, err) + } + if size < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return size, nil +} + +func parseNonNegativeDuration(field, value string) (time.Duration, error) { + d, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid %s %q: %w", field, value, err) + } + if d < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return d, nil +} + +// validateGitUpstreams checks upstream.git names, URLs and repository +// patterns. An upstream that upstream.auth has credentials for must list its +// repositories, so configuring a token does not open every repository it can +// read to everyone who can reach the proxy. +func (u *UpstreamConfig) validateGitUpstreams() error { + for name, upstream := range u.Git { + field := "upstream.git." + name + if name == "" || name == "." || name == ".." || name[0] == '.' || strings.ContainsAny(name, `/\\`) { + return fmt.Errorf("invalid upstream.git name %q", name) + } + if err := validateAbsoluteURL(field, upstream.URL); err != nil { + return err + } + if scheme, _, _ := strings.Cut(upstream.URL, "://"); !strings.EqualFold(scheme, "https") && !strings.EqualFold(scheme, "http") { + return fmt.Errorf("invalid %s %q: only http and https are supported", field, upstream.URL) + } + for _, pattern := range upstream.Repos { + if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { + return fmt.Errorf("invalid %s.repos pattern %q", field, pattern) + } + } + if len(upstream.Repos) == 0 && u.HasAuthWithin(upstream.URL) { + return fmt.Errorf("%s needs repos: upstream.auth has credentials for %s, "+ + "and everyone who can reach the proxy could read every repository they allow", field, upstream.URL) + } + } + return nil +} + +// HasAuthWithin reports whether an upstream.auth scope covers baseURL or a +// path under it. +func (u *UpstreamConfig) HasAuthWithin(baseURL string) bool { + base, err := parseAuthURL(baseURL) + if err != nil { + return false + } + for scope := range u.Auth { + configured, err := parseAuthURL(scope) + if err != nil { + continue + } + if authURLMatches(configured, base) || authURLMatches(base, configured) { + return true + } + } + return false +} diff --git a/internal/config/git_test.go b/internal/config/git_test.go new file mode 100644 index 00000000..6c9b47bd --- /dev/null +++ b/internal/config/git_test.go @@ -0,0 +1,172 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestGitUpstreamAcceptsURLOrObject(t *testing.T) { + want := map[string]GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/*"}}, + } + + var fromYAML struct { + Git map[string]GitUpstream `yaml:"git"` + } + err := yaml.Unmarshal([]byte(` +git: + github: "https://github.com" + wikimedia: + url: "https://gitlab.wikimedia.org" + repos: ["mediawiki/*"] +`), &fromYAML) + if err != nil { + t.Fatalf("YAML: %v", err) + } + + var fromJSON struct { + Git map[string]GitUpstream `json:"git"` + } + err = json.Unmarshal([]byte(`{"git": { + "github": "https://github.com", + "wikimedia": {"url": "https://gitlab.wikimedia.org", "repos": ["mediawiki/*"]} + }}`), &fromJSON) + if err != nil { + t.Fatalf("JSON: %v", err) + } + + for format, got := range map[string]map[string]GitUpstream{"YAML": fromYAML.Git, "JSON": fromJSON.Git} { + for name, w := range want { + if g := got[name]; g.URL != w.URL || !slices.Equal(g.Repos, w.Repos) { + t.Errorf("%s %s = %+v, want %+v", format, name, g, w) + } + } + } +} + +func TestGitUpstreamAllowsRepo(t *testing.T) { + open := GitUpstream{URL: "https://github.com"} + if !open.AllowsRepo("any/repo") { + t.Error("an upstream without repos refused a repository") + } + listed := GitUpstream{URL: "https://gitlab.example", Repos: []string{"mediawiki/*", "tools/one", "group/**"}} + for repo, want := range map[string]bool{ + "mediawiki/core": true, + "tools/one": true, + "tools/two": false, + "mediawiki/extensions/Echo": false, + "group/sub/project": true, + "group/project": true, + "group": false, + "groups/project": false, + } { + if got := listed.AllowsRepo(repo); got != want { + t.Errorf("AllowsRepo(%q) = %v, want %v", repo, got, want) + } + } +} + +func TestValidateGitUpstreams(t *testing.T) { + tests := []struct { + name string + upstream UpstreamConfig + wantErr bool + }{ + {name: "public", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}}}, + {name: "relative URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "github.com"}}}, wantErr: true}, + {name: "name with slash", upstream: UpstreamConfig{Git: map[string]GitUpstream{"a/b": {URL: "https://github.com"}}}, wantErr: true}, + {name: "name with leading dot", upstream: UpstreamConfig{Git: map[string]GitUpstream{".tmp": {URL: "https://github.com"}}}, wantErr: true}, + {name: "bad pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com", Repos: []string{"["}}}}, wantErr: true}, + {name: "nested pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"gerrit": {URL: "https://gerrit.example/r", Repos: []string{"mediawiki/**"}}}}}, + {name: "ssh URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "ssh://git@github.com"}}}, wantErr: true}, + { + name: "credentials without repos", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, + Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials for a path under the upstream", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, + Auth: map[string]AuthConfig{"https://git.example/team": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials with repos", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example", Repos: []string{"team/*"}}}, + Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, + }, + }, + { + name: "credentials for another host", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}, + Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGitConfigDefaults(t *testing.T) { + var g GitConfig + if err := g.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if g.CacheDir != defaultGitCacheDir || g.MaxSizeBytes() != 0 || g.MaxRepoSizeBytes() != 5<<30 || + g.RefreshWaitDuration() != 5*time.Second || g.StaleMaxAgeDuration() != 24*time.Hour { + t.Errorf("defaults = %+v", g) + } + + for _, bad := range []GitConfig{{MaxSize: "lots"}, {RefreshWait: "soon"}, {StaleMaxAge: "-1h"}} { + if err := bad.Validate(); err == nil { + t.Errorf("Validate(%+v) accepted it", bad) + } + } +} + +func TestLoadGitConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte(` +upstream: + git: + github: https://github.com +git: + cache_dir: /data/git + max_size: 50GB +`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + t.Setenv("PROXY_GIT_REFRESH_WAIT", "2s") + cfg.LoadFromEnv() + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if cfg.Upstream.Git["github"].URL != "https://github.com" || cfg.Git.CacheDir != "/data/git" || + cfg.Git.MaxSizeBytes() != 50<<30 || cfg.Git.RefreshWaitDuration() != 2*time.Second { + t.Errorf("loaded %+v and %+v", cfg.Upstream.Git, cfg.Git) + } +} diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go new file mode 100644 index 00000000..eee4d34b --- /dev/null +++ b/internal/gitcache/cache.go @@ -0,0 +1,421 @@ +// Package gitcache serves Git repositories read-only over smart HTTP from a +// local cache of bare repositories, refreshed from configured upstreams. +package gitcache + +import ( + "cmp" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + // tmpDirName holds clones in progress and evicted repositories being + // deleted. Upstream names cannot start with a dot, so it cannot clash + // with an upstream's directory, and the name is specific in case + // cache_dir is shared with other data. + tmpDirName = ".gitcache-tmp" + // usedStampInterval limits how often a repository's last use is written + // to disk. + usedStampInterval = time.Minute + // defaultLowSpeedTime is how many seconds a git transfer may receive + // nothing before it is abandoned. Fetches run with --quiet, so the + // upstream sends nothing while it counts objects; this is far longer. + defaultLowSpeedTime = 600 + ecosystem = "git" + upstreamHashLen = 8 + checkedStamp = "proxy-checked" + usedStamp = "proxy-used" + dirPermissions = 0o750 +) + +// Options configures a Cache. +type Options struct { + CacheDir string + MaxSize int64 + MaxRepoSize int64 + RefreshWait time.Duration + StaleMaxAge time.Duration + // Client makes every upstream request, including those of the git + // processes the cache runs. Redirects are never followed. + Client *http.Client + Logger *slog.Logger + // Upstreams maps each configured upstream's name to its directory in + // CacheDir, as UpstreamDir names it. Only those directories are loaded + // at startup. A directory UpstreamDir would name for one of these names + // at another URL, left by repointing the upstream, is removed. Nothing + // else in CacheDir is touched, so it may hold other data. + Upstreams map[string]string + + // refCheckTimeout and lowSpeedTime override their defaults in tests. + refCheckTimeout time.Duration + lowSpeedTime int +} + +// Cache is the Git repository cache. internal/handler serves it at /git. +type Cache struct { + opts Options + client *http.Client + logger *slog.Logger + gitPath string + dir string + relay *relay + fetches chan struct{} + refCheckTimeout time.Duration + lowSpeedTime int + + ctx context.Context + cancel context.CancelFunc + // refreshes counts the refreshes running, which Close waits for. + refreshes sync.WaitGroup + + mu sync.Mutex + // closed stops refreshes from starting once Close has begun. + closed bool + repos map[string]*repo + tooLarge map[string]time.Time + // failedWants records, by repository key and object ID, wants the + // upstream could not provide. + failedWants map[string]time.Time +} + +// repo is a cached repository, or one being cloned. users, refreshing and +// the map entry itself are guarded by Cache.mu. +type repo struct { + key string + dir string + + users int + refreshing *refreshCall + // writes serializes git fetches into the repository. + writes chan struct{} + + size atomic.Int64 + lastUsed atomic.Int64 + lastChecked atomic.Int64 + usedWritten atomic.Int64 +} + +// maxConcurrentFetches caps clones and fetches running at once. Refreshes +// outlive the requests that start them, so without a cap a client could +// start any number and hang up. +const maxConcurrentFetches = 16 + +// New creates the cache, loading repositories already in CacheDir. +func New(opts Options) (*Cache, error) { + gitPath, err := exec.LookPath("git") + if err != nil { + return nil, fmt.Errorf("upstream.git needs git installed: %w", err) + } + dir, err := filepath.Abs(opts.CacheDir) + if err != nil { + return nil, err + } + logger := opts.Logger + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + client := *opts.Client + client.Timeout = 0 + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + + c := &Cache{ + opts: opts, + client: &client, + logger: logger, + gitPath: gitPath, + dir: dir, + fetches: make(chan struct{}, maxConcurrentFetches), + refCheckTimeout: defaultRefCheckTimeout, + lowSpeedTime: defaultLowSpeedTime, + repos: make(map[string]*repo), + tooLarge: make(map[string]time.Time), + failedWants: make(map[string]time.Time), + } + if opts.refCheckTimeout > 0 { + c.refCheckTimeout = opts.refCheckTimeout + } + if opts.lowSpeedTime > 0 { + c.lowSpeedTime = opts.lowSpeedTime + } + if err := c.load(); err != nil { + return nil, err + } + if c.relay, err = newRelay(c.client, logger); err != nil { + return nil, fmt.Errorf("starting git relay: %w", err) + } + c.ctx, c.cancel = context.WithCancel(context.Background()) + // Applies a lowered git.max_size straight away. + c.evict() + return c, nil +} + +// Close stops the refreshes in progress and waits for them to finish, so +// none is still writing to the cache when it returns, then stops the relay. +// Fetches by ID run on request goroutines; stop the HTTP server first. +// Closing again does nothing. +func (c *Cache) Close() error { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return nil + } + c.closed = true + c.mu.Unlock() + c.cancel() + c.refreshes.Wait() + return c.relay.close() +} + +// UpstreamDir names the top-level cache directory of the upstream called +// name at baseURL. It includes a hash of the URL, so repointing an upstream +// starts a fresh cache rather than serving the old host's repositories. +func UpstreamDir(name, baseURL string) string { + digest := sha256.Sum256([]byte(strings.TrimSuffix(baseURL, "/"))) + return name + "-" + hex.EncodeToString(digest[:])[:upstreamHashLen] +} + +func (c *Cache) tmpDir() string { + return filepath.Join(c.dir, tmpDirName) +} + +// load clears what an earlier run left in the temporary directory and +// registers the repositories already cached. +func (c *Cache) load() error { + if err := os.RemoveAll(c.tmpDir()); err != nil { + return fmt.Errorf("clearing %s: %w", c.tmpDir(), err) + } + if err := os.MkdirAll(c.tmpDir(), dirPermissions); err != nil { + return err + } + c.removeRepointed() + for _, upstreamDir := range c.opts.Upstreams { + root := filepath.Join(c.dir, upstreamDir) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() || !strings.HasSuffix(d.Name(), ".git") || path == root { + return err + } + rel, _ := filepath.Rel(c.dir, path) + rp := newRepo(filepath.ToSlash(rel), path) + rp.size.Store(dirSize(path)) + rp.lastChecked.Store(stampTime(path, checkedStamp)) + rp.lastUsed.Store(stampTime(path, usedStamp)) + rp.usedWritten.Store(rp.lastUsed.Load()) + c.repos[rp.key] = rp + return filepath.SkipDir + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + return nil +} + +// upstreamDirPattern matches a directory UpstreamDir names, capturing the +// upstream's name. +var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`) + +// removeRepointed removes the directories of configured upstreams at URLs +// they no longer have. Each is moved aside first, so nothing can be read +// from it half deleted. Directories of upstreams no longer configured are +// only logged: nothing else says they are the cache's. +func (c *Cache) removeRepointed() { + entries, err := os.ReadDir(c.dir) + if err != nil { + c.logger.Warn("git cache: cannot list the cache directory", "error", err) + return + } + current := make(map[string]bool, len(c.opts.Upstreams)) + for _, upstreamDir := range c.opts.Upstreams { + current[upstreamDir] = true + } + var unconfigured []string + for _, entry := range entries { + name := entry.Name() + match := upstreamDirPattern.FindStringSubmatch(name) + if !entry.IsDir() || match == nil || current[name] { + continue + } + if _, configured := c.opts.Upstreams[match[1]]; !configured { + unconfigured = append(unconfigured, name) + continue + } + trash, err := os.MkdirTemp(c.tmpDir(), "repointed-") + if err == nil { + err = os.Rename(filepath.Join(c.dir, name), filepath.Join(trash, name)) + } + if err != nil { + c.logger.Warn("git cache: failed to remove the cache of a repointed upstream", "directory", name, "error", err) + continue + } + c.logger.Info("git cache: removed the cache of a repointed upstream", "directory", name) + if err := os.RemoveAll(trash); err != nil { + c.logger.Warn("git cache: failed to delete a removed cache", "path", trash, "error", err) + } + } + if len(unconfigured) > 0 { + c.logger.Info("git cache: leaving directories that look like caches of upstreams no longer configured; delete them if they are", + "directories", unconfigured) + } +} + +// acquire returns the repository for key, counting the caller as a user +// until release, so it is not evicted while in use. +func (c *Cache) acquire(key string) *repo { + c.mu.Lock() + defer c.mu.Unlock() + rp := c.repos[key] + if rp == nil { + rp = newRepo(key, filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[key] = rp + } + rp.users++ + return rp +} + +func (c *Cache) release(rp *repo) { + now := time.Now() + rp.lastUsed.Store(now.UnixNano()) + exists := rp.exists() + if exists && now.Sub(time.Unix(0, rp.usedWritten.Load())) >= usedStampInterval { + rp.usedWritten.Store(now.UnixNano()) + touch(rp.dir, usedStamp, now) + } + + c.mu.Lock() + rp.users-- + idle := rp.users == 0 && c.repos[rp.key] == rp + // Forget repositories that never got cloned, such as ones missing upstream. + if idle && !exists { + delete(c.repos, rp.key) + } + _, tooLarge := c.tooLarge[rp.key] + c.mu.Unlock() + if idle && exists && tooLarge { + c.evict() + } +} + +func newRepo(key, dir string) *repo { + return &repo{key: key, dir: dir, writes: make(chan struct{}, 1)} +} + +// lockWrites waits for other fetches into the repository to finish. +func (rp *repo) lockWrites(ctx context.Context) (func(), error) { + select { + case rp.writes <- struct{}{}: + return func() { <-rp.writes }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (rp *repo) exists() bool { + _, err := os.Stat(rp.dir) + return err == nil +} + +// evict removes repositories nobody is using: the least recently used until +// the cache fits in MaxSize, and any that could not be refreshed within +// MaxRepoSize, so the next request clones them again with only their +// branches and tags. It runs after every fetch, since only fetches grow the +// cache, and when a repository too large to refresh is released. A +// repository is moved aside under the lock, so a request that comes after it +// clones it again instead of reading it while it is deleted. +func (c *Cache) evict() { + c.mu.Lock() + var total int64 + idle := make([]*repo, 0, len(c.repos)) + for _, rp := range c.repos { + total += rp.size.Load() + if rp.users == 0 { + idle = append(idle, rp) + } + } + slices.SortFunc(idle, func(a, b *repo) int { + return cmp.Compare(a.lastUsed.Load(), b.lastUsed.Load()) + }) + var removed []string + for _, rp := range idle { + _, marked := c.tooLarge[rp.key] + tooLarge := marked && rp.exists() + if !tooLarge && (c.opts.MaxSize <= 0 || total <= c.opts.MaxSize) { + continue + } + trash, err := os.MkdirTemp(c.tmpDir(), "evicted-") + if err == nil { + err = os.Rename(rp.dir, filepath.Join(trash, "repo")) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + c.logger.Warn("git cache: failed to evict repository", "repository", rp.key, "error", err) + continue + } + delete(c.repos, rp.key) + total -= rp.size.Load() + removed = append(removed, trash) + reason := "lru" + if tooLarge { + // The clone that replaces it enforces the limit itself. + delete(c.tooLarge, rp.key) + reason = "too_large" + } + metrics.RecordArtifactEvicted(reason, ecosystem) + c.logger.Info("git cache: evicted repository", "repository", rp.key, "size", rp.size.Load(), "reason", reason) + } + c.mu.Unlock() + + for _, trash := range removed { + if err := os.RemoveAll(trash); err != nil { + c.logger.Warn("git cache: failed to delete evicted repository", "path", trash, "error", err) + } + } +} + +func dirSize(dir string) int64 { + var size int64 + _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err == nil && d.Type().IsRegular() { + if info, err := d.Info(); err == nil { + size += info.Size() + } + } + return nil + }) + return size +} + +// Stamps are empty files whose modification time records an event, so it +// survives restarts. +func stampTime(dir, name string) int64 { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + return 0 + } + return info.ModTime().UnixNano() +} + +func touch(dir, name string, t time.Time) { + path := filepath.Join(dir, name) + if err := os.Chtimes(path, t, t); errors.Is(err, fs.ErrNotExist) { + if f, err := os.Create(path); err == nil { + _ = f.Close() + _ = os.Chtimes(path, t, t) + } + } +} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go new file mode 100644 index 00000000..f5d14b2b --- /dev/null +++ b/internal/gitcache/cache_test.go @@ -0,0 +1,659 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/safehttp" +) + +func TestCloneMatchesUpstream(t *testing.T) { + for _, version := range []string{"0", "1", "2"} { + t.Run("protocol v"+version, func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + clone := tc.clone(t, "repo.git", "-c", "protocol.version="+version) + + want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) + if got := runGit(t, "", "-c", "protocol.version="+version, "ls-remote", "--heads", "--tags", tc.url("repo.git")); got != want { + t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) + } + if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { + t.Errorf("HEAD = %s, want %s", got, want) + } + runGit(t, clone, "fsck", "--no-dangling", "--no-progress") + }) + } +} + +func TestUpstreamWithoutProtocolV2(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.v0Only = true + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + clone := tc.clone(t, "repo") + if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { + t.Errorf("HEAD = %s, want %s", got, want) + } +} + +func TestUnchangedRepositoryIsNotFetchedAgain(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + tc.clone(t, "repo") + fetches := upstream.gitRequests.Load() + tc.clone(t, "repo") + if got := upstream.gitRequests.Load(); got != fetches { + t.Errorf("second clone made %d upstream fetch requests, want none", got-fetches) + } +} + +func TestPushedCommitIsServedOnNextClone(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + existing := tc.clone(t, "repo") + + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("clone after push: HEAD = %s, want %s", got, head) + } + runGit(t, existing, "pull", "--quiet", "--ff-only") + if got := runGit(t, existing, "rev-parse", "HEAD"); got != head { + t.Errorf("pull after push: HEAD = %s, want %s", got, head) + } + if got := runGit(t, tc.clone(t, "repo", "--depth=1"), "rev-list", "--count", "HEAD"); got != "1" { + t.Errorf("shallow clone has %s commits, want 1", got) + } +} + +func TestOnlyBranchesAndTagsAreCachedAndPruned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, work, "push", "--quiet", "origin", "main:refs/changes/01/1/1", "main:refs/heads/gone", "main:refs/tags/gone") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + refs := tc.cachedRefs(t) + if strings.Contains(refs, "refs/changes/") { + t.Errorf("cached refs/changes:\n%s", refs) + } + if !strings.Contains(refs, "refs/heads/gone") || !strings.Contains(refs, "refs/tags/gone") { + t.Fatalf("branch and tag not cached:\n%s", refs) + } + + runGit(t, work, "push", "--quiet", "origin", ":refs/heads/gone", ":refs/tags/gone") + tc.clone(t, "repo") + if refs := tc.cachedRefs(t); strings.Contains(refs, "gone") { + t.Errorf("deleted branch or tag still cached:\n%s", refs) + } + + // Gerrit advertises a ref per patch set, so asking for every ref would + // list hundreds of thousands for a large repository. + bodies := upstream.lsRefsBodies() + if len(bodies) == 0 { + t.Fatal("no ls-refs request reached the upstream") + } + for _, body := range bodies { + if !strings.Contains(body, "ref-prefix refs/heads/") || !strings.Contains(body, "ref-prefix refs/tags/") { + t.Errorf("ls-refs asked for every ref, want branches and tags only:\n%q", body) + } + } +} + +func TestDefaultBranchChangeIsFollowed(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + runGit(t, work, "checkout", "--quiet", "-b", "trunk") + head := upstream.commit(work, "on trunk") + runGit(t, work, "push", "--quiet", "origin", "trunk") + runGit(t, upstream.bare(), "symbolic-ref", "HEAD", "refs/heads/trunk") + + clone := tc.clone(t, "repo") + if got := runGit(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want trunk's %s", got, head) + } +} + +func TestConcurrentColdClonesShareOneFetch(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + release := upstream.holdRequests() + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + dir := filepath.Join(t.TempDir(), "clone") + if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { + t.Errorf("clone: %v\n%s", err, out) + } + }) + } + time.Sleep(200 * time.Millisecond) + release() + wg.Wait() + + if got := upstream.gitSessions.Load(); got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } +} + +func TestMissingRepositoryLeavesNothingBehind(t *testing.T) { + upstream := newFakeUpstream(t) + tc := newTestCache(t, upstream.server.URL, nil) + + resp, err := http.Get(tc.url("missing") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("status %d, want 404", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if _, err := os.Stat(filepath.Join(tc.dir, "up", "missing.git")); !os.IsNotExist(err) { + t.Errorf("repository directory left behind: %v", err) + } +} + +func TestStaleCopyServedOnlyWithinStaleMaxAge(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusServiceUnavailable) + tc.clone(t, "repo") + + waitIdle(t, tc.Cache) + rp := tc.acquire("up/repo.git") + rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) + tc.release(rp) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served a copy older than stale_max_age:\n%s", out) + } +} + +func TestRefusedRefreshIsNotServedStale(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusMovedPermanently} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(int32(status)) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served the cached copy after status %d:\n%s", status, out) + } + }) + } +} + +func TestSlowUpstreamServesCachedCopyAfterRefreshWait(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + release := upstream.holdRequests() + defer release() + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { + t.Errorf("HEAD = %s, want the cached %s", got, old) + } + release() + waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) +} + +func TestUpstreamCredentialsReachEveryUpstreamRequest(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.requireAuth = "Bearer secret" + upstream.newRepo("repo") + client := &http.Client{Transport: headerTransport{"Authorization", "Bearer secret"}} + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) + + tc.clone(t, "repo") + if got := upstream.unauthed.Load(); got != 0 { + t.Errorf("%d upstream requests lacked credentials", got) + } + if upstream.gitRequests.Load() == 0 { + t.Error("no fetch went through the relay") + } +} + +type headerTransport struct{ name, value string } + +func (h headerTransport) RoundTrip(r *http.Request) (*http.Response, error) { + r = r.Clone(r.Context()) + r.Header.Set(h.name, h.value) + return http.DefaultTransport.RoundTrip(r) +} + +// TestUpstreamAddressChecksApplyToGit uses a client that refuses loopback +// addresses, as the proxy's does by default. git must not reach the loopback +// upstream around it, whether the ref check or only git's fetch is refused. +func TestUpstreamAddressChecksApplyToGit(t *testing.T) { + blocked := safehttp.New(nil, safehttp.Options{}) + for name, client := range map[string]*http.Client{ + "every request": blocked, + "git's fetch": {Transport: gitOnlyTransport{blocked.Transport}}, + } { + t.Run(name, func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) + + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("clone through a blocked upstream succeeded:\n%s", out) + } + if got := upstream.gitRequests.Load(); got != 0 { + t.Errorf("git reached the upstream with %d requests", got) + } + }) + } +} + +// gitOnlyTransport sends the cache's own ref checks with the default +// transport, and git's relayed requests with next. +type gitOnlyTransport struct{ next http.RoundTripper } + +func (g gitOnlyTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if r.UserAgent() == gitUserAgent { + return http.DefaultTransport.RoundTrip(r) + } + return g.next.RoundTrip(r) +} + +// TestRedirectIsNotFollowed redirects the ref check, then only git's fetch. +func TestRedirectIsNotFollowed(t *testing.T) { + for _, redirectRefCheck := range []bool{true, false} { + t.Run(fmt.Sprint("ref check redirected: ", redirectRefCheck), func(t *testing.T) { + target := newFakeUpstream(t) + target.newRepo("repo") + targetURL, _ := url.Parse(target.server.URL) + passThrough := httputil.NewSingleHostReverseProxy(targetURL) + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() == gitUserAgent && !redirectRefCheck { + passThrough.ServeHTTP(w, r) + return + } + http.Redirect(w, r, target.server.URL+r.URL.RequestURI(), http.StatusFound) + })) + defer redirect.Close() + tc := newTestCache(t, redirect.URL, nil) + + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("followed a redirect:\n%s", out) + } + if got := target.gitRequests.Load(); got != 0 { + t.Errorf("redirect target received %d requests from git", got) + } + }) + } +} + +func TestRepositoryOverMaxRepoSizeIsRefused(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + big := make([]byte, 256<<10) + _, _ = rand.Read(big) + if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "big.bin") + runGit(t, work, "commit", "--quiet", "-m", "big") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = 64 << 10 }) + + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusInsufficientStorage { + t.Errorf("status %d, want 507", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { + t.Errorf("left %d entries in the temporary directory", len(entries)) + } + + requests := upstream.requests.Load() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("second attempt succeeded:\n%s", out) + } + if got := upstream.requests.Load(); got != requests { + t.Errorf("second attempt made %d upstream requests, want none", got-requests) + } +} + +// TestEvictionRemovesLeastRecentlyUsedIdleRepositories makes the least +// recently used repository one a request is reading, so the next one is +// evicted instead. +func TestEvictionRemovesLeastRecentlyUsedIdleRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + for _, name := range []string{"old", "busy", "new"} { + upstream.newRepo(name) + } + size := repoSize(t, upstream, "new") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = 2*size + size/2 }) + + tc.clone(t, "busy") + busy := tc.acquire("up/busy.git") + defer tc.release(busy) + busy.lastUsed.Store(0) + tc.clone(t, "old") + waitFor(t, func() bool { return tc.usersOf("up/old.git") == 0 }) + tc.repo("up/old.git").lastUsed.Store(1) + + // The third clone takes the cache past MaxSize. + tc.clone(t, "new") + waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "old.git")) }) + for _, name := range []string{"busy", "new"} { + if !fileExists(filepath.Join(tc.dir, "up", name+".git")) { + t.Errorf("%s was evicted", name) + } + } + tc.clone(t, "old") +} + +// repoSize is the size a cache gives repository name once cloned. +func repoSize(t *testing.T, upstream *fakeUpstream, name string) int64 { + t.Helper() + probe := newTestCache(t, upstream.server.URL, nil) + probe.clone(t, name) + return probe.repo("up/" + name + ".git").size.Load() +} + +func TestNewLoadsCachedRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if err := os.MkdirAll(filepath.Join(tc.tmpDir(), "clone-leftover"), 0o750); err != nil { + t.Fatal(err) + } + + reloaded, err := New(Options{ + Upstreams: map[string]string{"up": "up"}, + CacheDir: tc.dir, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reloaded.Close() }() + + rp := reloaded.repos["up/repo.git"] // nothing else uses reloaded yet + if rp == nil || rp.size.Load() == 0 || rp.lastChecked.Load() == 0 { + t.Fatalf("repository not loaded: %+v", rp) + } + if entries, _ := os.ReadDir(reloaded.tmpDir()); len(entries) != 0 { + t.Errorf("temporary directory not cleared: %v", entries) + } +} + +func waitForRepos(t *testing.T, c *Cache, n int) { + t.Helper() + waitFor(t, func() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.repos) == n + }) +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for !cond() { + select { + case <-ctx.Done(): + t.Fatal("condition not met in time") + case <-time.After(20 * time.Millisecond): + } + } +} + +// TestRewoundCacheIsBroughtBackInLine changes the cache behind the upstream's +// back, so it only matches again if the next request refreshes branches and +// tags and prunes what the upstream does not have. +func TestRewoundCacheIsBroughtBackInLine(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + cached := filepath.Join(tc.dir, "up", "repo.git") + runGit(t, cached, "update-ref", "refs/heads/main", "main~1") + runGit(t, cached, "update-ref", "refs/heads/extra", "main") + runGit(t, cached, "tag", "extra", "main") + + want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) + if got := runGit(t, "", "ls-remote", "--heads", "--tags", tc.url("repo")); got != want { + t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) + } +} + +// TestConcurrentWarmClonesDuringRefresh serves clones from the cache while a +// fetch writes to it. +func TestConcurrentWarmClonesDuringRefresh(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = time.Millisecond }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + dir := filepath.Join(t.TempDir(), "clone") + if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { + t.Errorf("clone: %v\n%s", err, out) + return + } + if got := runGit(t, dir, "rev-parse", "HEAD"); got != old && got != head { + t.Errorf("HEAD = %s, want %s or %s", got, old, head) + } + runGit(t, dir, "fsck", "--no-dangling", "--no-progress") + }) + } + wg.Wait() + waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) +} + +// TestStalledRefCheckFailsInsteadOfHanging has the upstream send headers and +// then nothing, as a broken load balancer might. +func TestStalledRefCheckFailsInsteadOfHanging(t *testing.T) { + stall := make(chan struct{}) + defer close(stall) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", advertisementType) + w.WriteHeader(http.StatusOK) + _ = http.NewResponseController(w).Flush() + select { + case <-stall: + case <-r.Context().Done(): + } + })) + defer upstream.Close() + tc := newTestCache(t, upstream.URL, func(o *Options) { o.refCheckTimeout = 200 * time.Millisecond }) + + start := time.Now() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("ls-remote succeeded:\n%s", out) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Errorf("took %s to fail", elapsed) + } +} + +// TestStalledFetchFailsInsteadOfHanging has the upstream answer the ref check +// but never git's fetch, as a connection that stalls mid-pack would. +func TestStalledFetchFailsInsteadOfHanging(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstream.stallFetches.Store(true) + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.lowSpeedTime = 1 }) + + start := time.Now() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("ls-remote succeeded:\n%s", out) + } + if elapsed := time.Since(start); elapsed > 20*time.Second { + t.Errorf("took %s to fail", elapsed) + } +} + +// TestFetchEvictsToMaxSize checks eviction runs on its own once a fetch has +// grown the cache past MaxSize. +func TestFetchEvictsToMaxSize(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("first") + upstream.newRepo("second") + size := repoSize(t, upstream, "first") + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) + tc.clone(t, "first") + waitIdle(t, tc.Cache) + tc.clone(t, "second") + waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "first.git")) }) + if !fileExists(filepath.Join(tc.dir, "up", "second.git")) { + t.Error("evicted the repository just fetched") + } +} + +// TestNewRemovesRepointedUpstreams starts a cache whose directory holds an +// upstream's cache at its old URL, and one of an upstream no longer +// configured. Only the first is known to be the cache's to remove. +func TestNewRemovesRepointedUpstreams(t *testing.T) { + dir := t.TempDir() + current := UpstreamDir("up", "https://new.example") + repointed := UpstreamDir("up", "https://old.example") + unconfigured := UpstreamDir("gone", "https://gone.example") + // Named after a configured upstream's name, but another upstream's. + longer := UpstreamDir("up-api", "https://api.example") + for _, name := range []string{current, repointed, unconfigured, longer} { + runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, name, "repo.git")) + } + c, err := New(Options{ + CacheDir: dir, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + Upstreams: map[string]string{"up": current}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + + for name, want := range map[string]bool{current: true, repointed: false, unconfigured: true, longer: true} { + if got := fileExists(filepath.Join(dir, name)); got != want { + t.Errorf("%s present = %v, want %v", name, got, want) + } + } + if keys := slices.Collect(maps.Keys(c.repos)); len(keys) != 1 || keys[0] != current+"/repo.git" { + t.Errorf("loaded %v, want only %s/repo.git", keys, current) + } +} + +// TestCacheDirSharedWithOtherData points the cache at a directory that also +// holds an artifact store and a database, as git.cache_dir: ./cache would. +func TestCacheDirSharedWithOtherData(t *testing.T) { + dir := t.TempDir() + _ = os.MkdirAll(filepath.Join(dir, "artifacts", "npm"), 0o750) + _ = os.WriteFile(filepath.Join(dir, "artifacts", "npm", "pkg.tgz"), []byte("x"), 0o600) + _ = os.WriteFile(filepath.Join(dir, "proxy.db"), []byte("db"), 0o600) + runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, "artifacts", "foo", "bar.git")) + c, err := New(Options{ + CacheDir: dir, + Client: &http.Client{}, + Upstreams: map[string]string{"github": UpstreamDir("github", "https://github.com")}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + for _, p := range []string{"artifacts/npm/pkg.tgz", "proxy.db", "artifacts/foo/bar.git"} { + if _, err := os.Stat(filepath.Join(dir, p)); err != nil { + t.Errorf("%s was deleted at startup: %v", p, err) + } + } + if len(c.repos) != 0 { + t.Errorf("loaded %v from outside the upstream directories", slices.Collect(maps.Keys(c.repos))) + } +} + +// TestCloseWaitsForRefreshes closes the cache while a clone is held up +// upstream. When Close returns, the clone must have finished, temporary +// directory included, so nothing is left writing to the cache. +func TestCloseWaitsForRefreshes(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstream.stallFetches.Store(true) + tc := newTestCache(t, upstream.server.URL, nil) + clones := func() []string { + names, _ := filepath.Glob(filepath.Join(tc.tmpDir(), "clone-*")) + return names + } + + go func() { _, _ = tryGit(t, "", "ls-remote", tc.url("repo")) }() + waitFor(t, func() bool { return len(clones()) > 0 }) + if err := tc.Close(); err != nil { + t.Fatal(err) + } + if left := clones(); len(left) != 0 { + t.Errorf("Close returned with the clone still running: %v", left) + } + + before := upstream.requests.Load() + if _, err := tc.Open(context.Background(), "up/other.git", upstream.server.URL+"/other", OpenOptions{}); !errors.Is(err, errClosed) { + t.Errorf("Open after Close: %v, want errClosed", err) + } + if got := upstream.requests.Load() - before; got != 0 { + t.Errorf("Open after Close made %d upstream requests", got) + } +} + +// TestCloseTwice checks closing again does nothing, as a cleanup that also +// closes the cache may. +func TestCloseTwice(t *testing.T) { + c, err := New(Options{CacheDir: t.TempDir(), Client: &http.Client{}}) + if err != nil { + t.Fatal(err) + } + if err := errors.Join(c.Close(), c.Close()); err != nil { + t.Errorf("closing twice: %v", err) + } +} diff --git a/internal/gitcache/git.go b/internal/gitcache/git.go new file mode 100644 index 00000000..736ee8a6 --- /dev/null +++ b/internal/gitcache/git.go @@ -0,0 +1,100 @@ +package gitcache + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "slices" + "strconv" + "strings" + "time" +) + +const ( + maxStderr = 4 << 10 + gitWaitDelay = 5 * time.Second +) + +// gitConfig applies to every git process the cache runs. +var gitConfig = [][2]string{ + // git would follow a redirect from the relay straight to the network. + {"http.followRedirects", "false"}, + {"protocol.version", "2"}, + // Auto gc stays on, as on any Git server, but runs inside the fetch + // rather than detaching, so no git process outlives the fetch that + // started it, and sizes are measured after it. + {"gc.autoDetach", "false"}, + {"maintenance.autoDetach", "false"}, + {"init.defaultBranch", "main"}, + // Keep every fetched pack as a pack. index-pack writes it to a + // temporary file and moves it into place only once complete, whereas a + // small pack is otherwise unpacked object by object, and a transfer cut + // off at max_repo_size would leave the objects before the cut, such as a + // commit without its tree's blobs. + {"transfer.unpackLimit", "1"}, + // Serve any commit the cache holds when a client asks for it by object + // ID, as the upstream would, including ones a force-push left behind, + // and do not advertise the refs that keep fetched ones. + {"uploadpack.allowAnySHA1InWant", "true"}, + {"uploadpack.hideRefs", wantedPrefix}, + // With http.lowSpeedTime, added per cache, gives up on a transfer that + // receives nothing for that long, so a connection that stalls mid-pack + // cannot hold a refresh for fetchTimeout. + {"http.lowSpeedLimit", "1"}, +} + +// command builds a git process with an environment of its own: none of the +// proxy's environment is passed on, so no system or user config, credential +// helper, HTTP proxy setting or GIT_ variable applies, and the only transport +// git may use is plain HTTP, which is what the relay speaks. +func (c *Cache) command(ctx context.Context, extraEnv []string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, c.gitPath, args...) + env := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + c.tmpDir(), + "LC_ALL=C", + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL=" + os.DevNull, + "GIT_TERMINAL_PROMPT=0", + "GIT_ALLOW_PROTOCOL=http", + } + config := slices.Concat(gitConfig, [][2]string{{"http.lowSpeedTime", strconv.Itoa(c.lowSpeedTime)}}) + env = append(env, "GIT_CONFIG_COUNT="+strconv.Itoa(len(config))) + for i, kv := range config { + env = append(env, + "GIT_CONFIG_KEY_"+strconv.Itoa(i)+"="+kv[0], + "GIT_CONFIG_VALUE_"+strconv.Itoa(i)+"="+kv[1]) + } + env = append(env, extraEnv...) + cmd.Env = env + cmd.WaitDelay = gitWaitDelay + return cmd +} + +// run runs git in the repository at dir and returns its standard output. +func (c *Cache) run(ctx context.Context, dir string, args ...string) (string, error) { + cmd := c.command(ctx, nil, append([]string{"-C", dir}, args...)...) + var stdout bytes.Buffer + stderr := &limitedBuffer{max: maxStderr} + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git %s: %w: %s", args[0], err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +// limitedBuffer keeps the first max bytes written to it. +type limitedBuffer struct { + bytes.Buffer + max int +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if room := b.max - b.Len(); room > 0 { + b.Buffer.Write(p[:min(len(p), room)]) + } + return len(p), nil +} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go new file mode 100644 index 00000000..ce6becae --- /dev/null +++ b/internal/gitcache/helpers_test.go @@ -0,0 +1,349 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "errors" + "io" + "log" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// gitEnv isolates the tests' own git commands from the machine's config. +func gitEnv(home string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } +} + +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := tryGit(t, dir, args...) + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(out) +} + +func tryGit(t *testing.T, dir string, args ...string) (string, error) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// httpBackendPath finds git http-backend, which serves the fake upstreams, +// and skips the test where git is installed without it. +func httpBackendPath(t *testing.T) string { + t.Helper() + dir := runGit(t, "", "--exec-path") + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(dir, name); fileExists(path) { + return path + } + } + t.Skip("git http-backend is not installed") + return "" +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// fakeUpstream serves bare repositories under root with git http-backend. +type fakeUpstream struct { + t *testing.T + root string + server *httptest.Server + + // status, when set, is returned instead of serving the request. + status atomic.Int32 + // requireAuth, when set, is the Authorization every request must carry. + requireAuth string + // v0Only drops Git-Protocol, so only protocol v0 is spoken. + v0Only bool + // hold, when set, blocks requests until it is closed. + holdMu sync.Mutex + hold chan struct{} + + requests atomic.Int32 + // gitRequests counts requests from git processes, which reach the + // upstream through the relay, and gitSessions the fetches they make. + gitRequests atomic.Int32 + gitSessions atomic.Int32 + unauthed atomic.Int32 + + // stallFetches makes git's own upload-pack requests hang. + stallFetches atomic.Bool + + bodiesMu sync.Mutex + bodies []string +} + +// recordBody keeps the decoded body of a POST. It also gives the request a +// Content-Length, since net/http/cgi refuses chunked bodies, which git sends +// once a request outgrows http.postBuffer. +func (u *fakeUpstream) recordBody(r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + u.t.Errorf("reading request body: %v", err) + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + r.ContentLength = int64(len(raw)) + r.TransferEncoding = nil + decoded := raw + if r.Header.Get("Content-Encoding") == "gzip" { + if zr, err := gzip.NewReader(bytes.NewReader(raw)); err == nil { + decoded, _ = io.ReadAll(zr) + } + } + u.bodiesMu.Lock() + u.bodies = append(u.bodies, string(decoded)) + u.bodiesMu.Unlock() +} + +// lsRefsBodies returns the ls-refs requests the upstream received. +func (u *fakeUpstream) lsRefsBodies() []string { + u.bodiesMu.Lock() + defer u.bodiesMu.Unlock() + var out []string + for _, body := range u.bodies { + if strings.Contains(body, "command=ls-refs") { + out = append(out, body) + } + } + return out +} + +func newFakeUpstream(t *testing.T) *fakeUpstream { + t.Helper() + u := &fakeUpstream{t: t, root: t.TempDir()} + backend := &cgi.Handler{ + Path: httpBackendPath(t), + Stderr: io.Discard, + Logger: log.New(io.Discard, "", 0), + Env: append(gitEnv(t.TempDir()), + "GIT_PROJECT_ROOT="+u.root, + "GIT_HTTP_EXPORT_ALL=1"), + } + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + if r.UserAgent() != gitUserAgent { + u.gitRequests.Add(1) + if r.Method == http.MethodGet { + u.gitSessions.Add(1) + } + } + u.holdMu.Lock() + hold := u.hold + u.holdMu.Unlock() + if hold != nil { + <-hold + } + if status := u.status.Load(); status != 0 { + http.Error(w, "unavailable", int(status)) + return + } + if u.requireAuth != "" && r.Header.Get("Authorization") != u.requireAuth { + u.unauthed.Add(1) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if u.v0Only { + r.Header.Del("Git-Protocol") + } + if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { + // The server notices the client leaving only once the body is read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + return + } + if r.Method == http.MethodPost { + u.recordBody(r) + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(u.server.Close) + return u +} + +func (u *fakeUpstream) holdRequests() func() { + u.holdMu.Lock() + defer u.holdMu.Unlock() + hold := make(chan struct{}) + u.hold = hold + var once sync.Once + return func() { + once.Do(func() { + u.holdMu.Lock() + u.hold = nil + u.holdMu.Unlock() + close(hold) + }) + } +} + +// newRepo creates {name}.git upstream with one commit on main and a tag, and +// returns a working clone to commit and push from. +func (u *fakeUpstream) newRepo(name string) string { + u.t.Helper() + bare := filepath.Join(u.root, name+".git") + runGit(u.t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := u.t.TempDir() + runGit(u.t, "", "clone", "--quiet", bare, work) + runGit(u.t, work, "checkout", "--quiet", "-b", "main") + u.commit(work, "first") + runGit(u.t, work, "tag", "-a", "-m", "v1", "v1") + runGit(u.t, work, "push", "--quiet", "origin", "main", "v1") + return work +} + +func (u *fakeUpstream) commit(work, msg string) string { + u.t.Helper() + if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte(msg+"\n"), 0o600); err != nil { + u.t.Fatal(err) + } + runGit(u.t, work, "add", "file.txt") + runGit(u.t, work, "commit", "--quiet", "-m", msg) + return runGit(u.t, work, "rev-parse", "HEAD") +} + +// waitIdle waits until no request or refresh is using a repository, so a +// test can inspect or change the cache without racing a request finishing. +func waitIdle(t *testing.T, c *Cache) { + t.Helper() + waitFor(t, func() bool { + c.mu.Lock() + defer c.mu.Unlock() + for _, rp := range c.repos { + if rp.users > 0 { + return false + } + } + return true + }) +} + +// bare is the upstream repository named "repo". +// repo returns the cache's entry for key, reading the map under the lock. +func (tc *testCache) repo(key string) *repo { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.repos[key] +} + +// usersOf is the number of requests and refreshes using the repository. +func (tc *testCache) usersOf(key string) int { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.repos[key].users +} + +func (u *fakeUpstream) bare() string { + return filepath.Join(u.root, "repo.git") +} + +type testCache struct { + *Cache + server *httptest.Server +} + +func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { + t.Helper() + opts := Options{ + Upstreams: map[string]string{"up": "up"}, + CacheDir: t.TempDir(), + RefreshWait: 5 * time.Second, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + } + if configure != nil { + configure(&opts) + } + c, err := New(opts) + if err != nil { + t.Fatalf("New: %v", err) + } + server := httptest.NewServer(testFront(c, upstreamURL, true)) + t.Cleanup(func() { + server.Close() + _ = c.Close() + }) + return &testCache{Cache: c, server: server} +} + +// testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as +// internal/handler does once it has validated the request, with repo +// fetched from upstreamURL/{repo}. fetchWants is OpenOptions.FetchWants. +func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/git/up/") + repo, infoRefs := strings.CutSuffix(path, "/info/refs") + if !infoRefs { + var ok bool + if repo, ok = strings.CutSuffix(path, "/git-upload-pack"); !ok || r.Method != http.MethodPost { + http.Error(w, "not served", http.StatusForbidden) + return + } + } else if r.URL.RawQuery != "service=git-upload-pack" { + http.Error(w, "not served", http.StatusForbidden) + return + } + lease, err := c.Open(r.Context(), "up/"+strings.TrimSuffix(repo, ".git")+".git", upstreamURL+"/"+repo, + OpenOptions{Refresh: infoRefs, FetchWants: fetchWants}) + switch { + case errors.Is(err, ErrNotFound): + http.Error(w, err.Error(), http.StatusNotFound) + return + case errors.Is(err, ErrTooLarge): + http.Error(w, err.Error(), http.StatusInsufficientStorage) + return + case err != nil: + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer lease.Release() + if infoRefs { + lease.ServeInfoRefs(w, r) + } else { + lease.ServeUploadPack(w, r) + } + }) +} + +func (tc *testCache) url(repo string) string { + return tc.server.URL + "/git/up/" + repo +} + +// clone clones repo through the cache with the given protocol version and +// returns the working copy. +func (tc *testCache) clone(t *testing.T, repo string, args ...string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "clone") + runGit(t, "", append(append([]string{"clone", "--quiet"}, args...), tc.url(repo), dir)...) + return dir +} + +// cachedRefs lists the refs of the cached copy of "repo". +func (tc *testCache) cachedRefs(t *testing.T) string { + t.Helper() + return runGit(t, filepath.Join(tc.dir, "up", "repo.git"), "for-each-ref", "--format=%(objectname) %(refname)") +} diff --git a/internal/gitcache/pktline.go b/internal/gitcache/pktline.go new file mode 100644 index 00000000..5f6095ba --- /dev/null +++ b/internal/gitcache/pktline.go @@ -0,0 +1,65 @@ +package gitcache + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +// pkt-line framing (gitprotocol-common): four hex digits giving the packet +// length including themselves, then the payload. The lengths 0000, 0001 and +// 0002 are the flush, delimiter and response-end packets. +const ( + pktFlush = "0000" + pktDelim = "0001" + pktHeaderLen = 4 + pktMaxLen = 65520 + pktSpecialLast = 2 +) + +type pktKind int + +const ( + pktData pktKind = iota + pktFlushKind + pktDelimKind + pktEndKind +) + +func pktLine(payload string) string { + return fmt.Sprintf("%04x%s", len(payload)+pktHeaderLen, payload) +} + +type pktReader struct { + r *bufio.Reader +} + +func newPktReader(r io.Reader) *pktReader { + return &pktReader{r: bufio.NewReader(r)} +} + +// next returns the next packet. A data packet's payload has its trailing +// newline removed. +func (p *pktReader) next() (pktKind, string, error) { + var header [pktHeaderLen]byte + if _, err := io.ReadFull(p.r, header[:]); err != nil { + return 0, "", err + } + n, err := strconv.ParseUint(string(header[:]), 16, 16) + if err != nil { + return 0, "", fmt.Errorf("invalid pkt-line length %q", header) + } + if n <= pktSpecialLast { + return pktKind(n + 1), "", nil + } + if n < pktHeaderLen || n > pktMaxLen { + return 0, "", fmt.Errorf("invalid pkt-line length %d", n) + } + payload := make([]byte, n-pktHeaderLen) + if _, err := io.ReadFull(p.r, payload); err != nil { + return 0, "", err + } + return pktData, strings.TrimSuffix(string(payload), "\n"), nil +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go new file mode 100644 index 00000000..801e8b3f --- /dev/null +++ b/internal/gitcache/refresh.go @@ -0,0 +1,336 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + // fetchTimeout bounds one refresh, including a cold clone of a large + // repository. + fetchTimeout = time.Hour + // defaultRefCheckTimeout bounds the ref check, so an upstream that + // stalls mid-response does not hold a repository's refresh for + // fetchTimeout. + defaultRefCheckTimeout = 2 * time.Minute + // tooLargeRetryAfter is how long a repository that passed + // git.max_repo_size is left alone before it is tried again. + tooLargeRetryAfter = time.Hour +) + +// errUpstreamUnanswered marks a fetch the upstream never answered, such as +// one during an outage. +var errUpstreamUnanswered = errors.New("upstream did not answer the fetch") + +// errClosed is returned for a refresh asked for once the cache is closing. +var errClosed = errors.New("git cache is closed") + +// refreshCall is a refresh in progress, shared by every request for the +// repository that arrives while it runs. +type refreshCall struct { + done chan struct{} + err error +} + +// ensureFresh brings the repository up to date with its upstream before it is +// served. A repository checked successfully within StaleMaxAge is served +// from the cache if the upstream fails or takes longer than RefreshWait; the +// refresh carries on in the background either way. One never cloned, or not +// checked within StaleMaxAge, is served only once the refresh succeeds. +func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error { + call := c.startRefresh(rp, repoURL) + checked := time.Unix(0, rp.lastChecked.Load()) + if !rp.exists() || time.Since(checked) > c.opts.StaleMaxAge { + select { + case <-call.done: + return call.err + case <-ctx.Done(): + return ctx.Err() + } + } + + timer := time.NewTimer(c.opts.RefreshWait) + defer timer.Stop() + select { + case <-call.done: + if call.err != nil && servesStale(call.err) { + c.logger.Warn("git cache: refresh failed, serving cached copy", + "repository", rp.key, "checked", checked, "error", call.err) + return nil + } + return call.err + case <-timer.C: + c.logger.Info("git cache: refresh still running, serving cached copy", + "repository", rp.key, "checked", checked) + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// servesStale reports whether the cached copy may be served after err: not +// when the upstream says the repository is gone or refuses access to it. +func servesStale(err error) bool { + return !errors.Is(err, ErrNotFound) && !errors.Is(err, ErrRefused) +} + +// startRefresh starts a refresh of the repository, or joins the one running. +// The refresh holds a use of the repository until it ends. +func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { + c.mu.Lock() + defer c.mu.Unlock() + if rp.refreshing != nil { + return rp.refreshing + } + call := &refreshCall{done: make(chan struct{})} + if c.closed { + call.err = errClosed + close(call.done) + return call + } + rp.refreshing = call + rp.users++ + c.refreshes.Add(1) + go func() { + defer c.refreshes.Done() + ctx, cancel := context.WithTimeout(c.ctx, fetchTimeout) + start := time.Now() + call.err = c.refresh(ctx, rp, repoURL) + cancel() + metrics.RecordUpstreamFetch(ecosystem, time.Since(start)) + // A refresh cut short by Close is not an upstream failure. + if call.err != nil && c.ctx.Err() == nil { + metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err}) + c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err) + } + c.mu.Lock() + rp.refreshing = nil + c.mu.Unlock() + c.release(rp) + close(call.done) + if call.err == nil { + c.evict() + } + }() + return call +} + +// refresh checks the upstream's branches and tags and fetches if they differ +// from the cached ones, cloning the repository if it is not cached yet. +func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string) error { + if c.isTooLarge(rp.key) { + return ErrTooLarge + } + checkCtx, cancel := context.WithTimeout(ctx, c.refCheckTimeout) + remote, err := c.listUpstreamRefs(checkCtx, repoURL) + cancel() + if err != nil { + return err + } + if rp.exists() { + err = c.update(ctx, rp, repoURL, remote) + } else { + err = c.clone(ctx, rp, repoURL, remote) + } + if errors.Is(err, ErrTooLarge) { + c.markTooLarge(rp.key) + } + if err != nil { + return err + } + now := time.Now() + rp.lastChecked.Store(now.UnixNano()) + touch(rp.dir, checkedStamp, now) + return nil +} + +// clone builds the repository in a temporary directory and moves it into +// place, so no request ever reads a partial clone. +func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { + tmp, err := os.MkdirTemp(c.tmpDir(), "clone-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmp) }() + + if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { + return err + } + if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, refreshFlags, branchesAndTags...); err != nil { + return err + } + if err := c.setHead(ctx, tmp, remote, ""); err != nil { + return err + } + size := dirSize(tmp) + if c.opts.MaxRepoSize > 0 && size > c.opts.MaxRepoSize { + return ErrTooLarge + } + if err := os.MkdirAll(filepath.Dir(rp.dir), dirPermissions); err != nil { + return err + } + if err := os.Rename(tmp, rp.dir); err != nil { + return fmt.Errorf("moving clone into place: %w", err) + } + rp.size.Store(size) + c.logger.Info("git cache: cloned repository", "repository", rp.key, "size", size) + return nil +} + +// update fetches when the upstream's branches or tags differ from the cached +// ones, and follows a change of the upstream's default branch. +func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + local, wanted, err := c.localRefs(ctx, rp.dir) + if err != nil { + return err + } + fetched := !maps.Equal(local, remote.refs) + if fetched { + limit, err := c.fetchLimit(rp) + if err != nil { + return err + } + err = c.fetch(ctx, rp.dir, repoURL, limit, refreshFlags, branchesAndTags...) + rp.size.Store(dirSize(rp.dir)) + if err != nil { + return err + } + } + if err := c.pruneWanted(ctx, rp.dir, wanted, fetched); err != nil { + c.logger.Warn("git cache: pruning wanted refs failed", "repository", rp.key, "error", err) + } + head, _ := c.run(ctx, rp.dir, "symbolic-ref", "--quiet", "HEAD") + return c.setHead(ctx, rp.dir, remote, strings.TrimSpace(head)) +} + +// branchesAndTags are the refspecs of a refresh. A bare clone has none of its +// own, and other refs, such as Gerrit's refs/changes, are left out. +var branchesAndTags = []string{"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"} + +// refreshFlags delete the branches and tags the upstream no longer has, and +// update all refs together, so a client served while a refresh runs never +// sees only some of them moved. +var refreshFlags = []string{"--prune", "--atomic"} + +// fetchLimit returns how many more bytes the repository may take, 0 for no +// limit. +func (c *Cache) fetchLimit(rp *repo) (int64, error) { + if c.opts.MaxRepoSize <= 0 { + return 0, nil + } + limit := c.opts.MaxRepoSize - rp.size.Load() + if limit <= 0 { + return 0, ErrTooLarge + } + return limit, nil +} + +// fetch fetches refspecs from the upstream into the repository at dir, +// through the relay, adding flags to git fetch. limit caps the bytes +// fetched, 0 for no cap. +func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, flags []string, refspecs ...string) error { + select { + case c.fetches <- struct{}{}: + defer func() { <-c.fetches }() + case <-ctx.Done(): + return ctx.Err() + } + + relayURL, route, unregister := c.relay.register(repoURL, limit) + defer unregister() + args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) + _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) + if err != nil { + // index-pack leaves its temporary pack behind when its input ends + // early. Fetches into a repository run one at a time, so any left + // now are this fetch's. + leftovers, _ := filepath.Glob(filepath.Join(dir, "objects", "pack", "tmp_*")) + for _, path := range leftovers { + _ = os.Remove(path) + } + } + if route.exceeded.Load() { + return ErrTooLarge + } + if err != nil && !route.answered.Load() { + return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) + } + return err +} + +// setHead points HEAD at the upstream's default branch, when the upstream +// names one and it is among the cached branches. +func (c *Cache) setHead(ctx context.Context, dir string, remote remoteRefs, current string) error { + if remote.head == "" || remote.head == current { + return nil + } + if _, ok := remote.refs[remote.head]; !ok { + return nil + } + _, err := c.run(ctx, dir, "symbolic-ref", "HEAD", remote.head) + return err +} + +// localRefs lists the cached branches and tags, and the names of the wanted +// refs. +func (c *Cache) localRefs(ctx context.Context, dir string) (map[string]string, []string, error) { + out, err := c.run(ctx, dir, "for-each-ref", "--format=%(objectname) %(refname)", headsPrefix, tagsPrefix, wantedPrefix) + if err != nil { + return nil, nil, err + } + refs := make(map[string]string) + var wanted []string + for line := range strings.Lines(out) { + oid, name, ok := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + switch { + case !ok: + case strings.HasPrefix(name, wantedPrefix): + wanted = append(wanted, name) + default: + refs[name] = oid + } + } + return refs, wanted, nil +} + +func errorType(err error) string { + switch { + case errors.Is(err, ErrNotFound): + return "not_found" + case errors.Is(err, ErrRefused): + return "refused" + case errors.Is(err, ErrTooLarge): + return "too_large" + } + return "fetch_failed" +} + +func (c *Cache) isTooLarge(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + since, ok := c.tooLarge[key] + if ok && time.Since(since) >= tooLargeRetryAfter { + delete(c.tooLarge, key) + return false + } + return ok +} + +func (c *Cache) markTooLarge(key string) { + c.mu.Lock() + defer c.mu.Unlock() + c.tooLarge[key] = time.Now() +} diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go new file mode 100644 index 00000000..437d7b55 --- /dev/null +++ b/internal/gitcache/relay.go @@ -0,0 +1,175 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io" + "log/slog" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + relayIDBytes = 16 + relayReadHeaderTimeout = 30 * time.Second +) + +// relay is a loopback HTTP server that every git fetch the cache runs goes +// through. git never reaches the network itself: each fetch gets a route to +// one upstream repository, and the relay forwards its requests with the +// proxy's upstream client, so address checks, upstream.auth, the access log +// and the no-redirect rule apply as they do to the proxy's own requests, and +// git never sees a credential. +type relay struct { + client *http.Client + logger *slog.Logger + listener net.Listener + server *http.Server + base string + + mu sync.Mutex + routes map[string]*relayRoute +} + +type relayRoute struct { + repoURL string + // limit caps the response bytes the route relays, 0 for no cap. + limit int64 + relayed atomic.Int64 + exceeded atomic.Bool + // answered records that the upstream answered a git-upload-pack + // request, so a failed fetch was not an outage. + answered atomic.Bool +} + +func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if listener, err = net.Listen("tcp", "[::1]:0"); err != nil { + return nil, err + } + } + rl := &relay{ + client: client, + logger: logger, + listener: listener, + base: "http://" + listener.Addr().String(), + routes: make(map[string]*relayRoute), + } + rl.server = &http.Server{Handler: rl, ReadHeaderTimeout: relayReadHeaderTimeout} + go func() { + if err := rl.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("git relay stopped", "error", err) + } + }() + return rl, nil +} + +func (rl *relay) close() error { + return rl.server.Close() +} + +// register adds a route to repoURL and returns the URL git fetches from and +// a function that removes the route. +func (rl *relay) register(repoURL string, limit int64) (string, *relayRoute, func()) { + var b [relayIDBytes]byte + _, _ = rand.Read(b[:]) + id := hex.EncodeToString(b[:]) + route := &relayRoute{repoURL: repoURL, limit: limit} + + rl.mu.Lock() + rl.routes[id] = route + rl.mu.Unlock() + + return rl.base + "/" + id, route, func() { + rl.mu.Lock() + delete(rl.routes, id) + rl.mu.Unlock() + } +} + +func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { + id, suffix, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + rl.mu.Lock() + route := rl.routes[id] + rl.mu.Unlock() + if route == nil { + http.NotFound(w, r) + return + } + + var target string + switch { + case suffix == "info/refs" && r.Method == http.MethodGet && r.URL.RawQuery == "service=git-upload-pack": + target = route.repoURL + "/info/refs?service=git-upload-pack" + case suffix == "git-upload-pack" && r.Method == http.MethodPost: + target = route.repoURL + "/git-upload-pack" + default: + http.NotFound(w, r) + return + } + + out, err := http.NewRequestWithContext(r.Context(), r.Method, target, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + out.ContentLength = r.ContentLength + for _, name := range []string{"Git-Protocol", "Content-Type", "Content-Encoding", "Accept", "User-Agent"} { + if value := r.Header.Get(name); value != "" { + out.Header.Set(name, value) + } + } + + resp, err := rl.client.Do(out) + if err != nil { + rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + // A redirect passed on would send git to the Location itself. + if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { + http.Error(w, "upstream redirect not followed", http.StatusBadGateway) + return + } + + if r.Method == http.MethodPost && resp.StatusCode == http.StatusOK { + route.answered.Store(true) + } + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, &routeReader{r: resp.Body, route: route}); err != nil { + if route.exceeded.Load() { + // Cut the connection so git fails rather than keeping a pack + // that merely looks short. + panic(http.ErrAbortHandler) + } + if !errors.Is(err, context.Canceled) { + rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) + } + } +} + +// routeReader counts relayed bytes against the route's limit. +type routeReader struct { + r io.Reader + route *relayRoute +} + +func (rr *routeReader) Read(p []byte) (int, error) { + n, err := rr.r.Read(p) + if total := rr.route.relayed.Add(int64(n)); rr.route.limit > 0 && total > rr.route.limit { + rr.route.exceeded.Store(true) + return 0, ErrTooLarge + } + return n, err +} diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go new file mode 100644 index 00000000..45726bcb --- /dev/null +++ b/internal/gitcache/relay_test.go @@ -0,0 +1,96 @@ +package gitcache + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func newTestRelay(t *testing.T) *relay { + t.Helper() + rl, err := newRelay(&http.Client{}, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rl.close() }) + return rl +} + +func TestRelayOnlyForwardsFetchRequests(t *testing.T) { + var forwarded atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + forwarded.Add(1) + })) + defer upstream.Close() + rl := newTestRelay(t) + base, _, unregister := rl.register(upstream.URL+"/repo", 0) + + refused := []struct{ method, url string }{ + {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, + {http.MethodGet, base + "/info/refs?service=git-receive-pack"}, + {http.MethodPost, base + "/git-receive-pack"}, + {http.MethodGet, base + "/HEAD"}, + {http.MethodGet, base + "/git-upload-pack"}, + } + for _, r := range refused { + req, _ := http.NewRequest(r.method, r.url, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s %s: status %d, want 404", r.method, r.url, resp.StatusCode) + } + } + if got := forwarded.Load(); got != 0 { + t.Errorf("forwarded %d refused requests", got) + } + + unregister() + resp, err := http.Get(base + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound || forwarded.Load() != 0 { + t.Errorf("unregistered route: status %d, forwarded %d", resp.StatusCode, forwarded.Load()) + } +} + +// TestRelayStopsAtLimit checks a fetch over git.max_repo_size is cut off +// rather than downloaded in full and thrown away. +func TestRelayStopsAtLimit(t *testing.T) { + const size, limit = 4 << 20, 64 << 10 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.Copy(w, io.LimitReader(zeros{}, size)) + })) + defer upstream.Close() + rl := newTestRelay(t) + base, route, unregister := rl.register(upstream.URL+"/repo", limit) + defer unregister() + + resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if err == nil { + t.Errorf("read %d bytes without an error", n) + } + if !route.exceeded.Load() || route.relayed.Load() > size/2 { + t.Errorf("exceeded %v after relaying %d of %d bytes", route.exceeded.Load(), route.relayed.Load(), size) + } +} + +type zeros struct{} + +func (zeros) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go new file mode 100644 index 00000000..b6fab6ea --- /dev/null +++ b/internal/gitcache/serve.go @@ -0,0 +1,151 @@ +package gitcache + +import ( + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "path/filepath" + "regexp" + "strings" +) + +// gitProtocolPattern is what a Git-Protocol header may hold, such as +// "version=2". It is passed to upload-pack in its environment. +var gitProtocolPattern = regexp.MustCompile(`^[A-Za-z0-9=:._-]*$`) + +var errInvalidKey = errors.New("invalid repository key") + +// Lease is a use of a cached repository. The repository is not evicted +// until the lease is released. +type Lease struct { + cache *Cache + repo *repo + repoURL string + hit bool + fetchWants bool +} + +// OpenOptions says what Open does besides returning the repository. +type OpenOptions struct { + // Refresh brings a cached repository up to date with the upstream first, + // or leaves it as it is when the upstream fails or is slow and the copy + // is recent enough (see Options). + Refresh bool + // FetchWants lets ServeUploadPack fetch commits a client asks for by ID + // that the cache lacks. Leave it off where a fetch by ID could reach + // commits the client may not read through this repository, such as + // GitHub forks with credentials configured. + FetchWants bool +} + +// Open returns a lease on the repository cached under key, a relative path +// such as github-1a2b3c4d/acme/widgets.git, whose upstream is repoURL. A +// repository not cached yet is cloned first. +func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) (*Lease, error) { + if !strings.HasSuffix(key, ".git") || !filepath.IsLocal(filepath.FromSlash(key)) { + return nil, errInvalidKey + } + rp := c.acquire(key) + hit := rp.exists() + if opts.Refresh || !hit { + if err := c.ensureFresh(ctx, rp, repoURL); err != nil { + c.release(rp) + return nil, err + } + } + return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil +} + +// Hit reports whether the repository was already cached when it was opened. +func (l *Lease) Hit() bool { + return l.hit +} + +// Release ends the lease. +func (l *Lease) Release() { + l.cache.release(l.repo) +} + +// ServeInfoRefs answers GET info/refs?service=git-upload-pack. +func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { + protocol := gitProtocol(r) + setNoCache(w) + w.Header().Set("Content-Type", advertisementType) + // Protocol v2 starts with its version line; earlier versions with the + // service name. + if !isProtocolV2(protocol) { + _, _ = io.WriteString(w, pktLine("# service=git-upload-pack\n")+pktFlush) + } + l.cache.uploadPack(w, r, l.repo, protocol, nil, "--advertise-refs") +} + +// ServeUploadPack answers POST git-upload-pack. With OpenOptions.FetchWants, +// commits the request wants that the cache lacks are fetched from the +// upstream first. +func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { + c := l.cache + body := io.Reader(r.Body) + if r.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(r.Body) + if err != nil { + http.Error(w, "invalid gzip body", http.StatusBadRequest) + return + } + defer func() { _ = gz.Close() }() + body = gz + } + if l.fetchWants { + var wants []string + wants, body = readWants(body) + if err := c.fetchMissingWants(r.Context(), l.repo, l.repoURL, wants); err != nil && r.Context().Err() == nil { + // upload-pack reports the missing objects to the client. + c.logger.Warn("git cache: fetching wanted objects failed", "repository", l.repo.key, "error", err) + } + } + setNoCache(w) + w.Header().Set("Content-Type", resultType) + c.uploadPack(w, r, l.repo, gitProtocol(r), body) +} + +// uploadPack runs git upload-pack on the cached repository in stateless mode, +// the way git http-backend does. +func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, protocol string, stdin io.Reader, args ...string) { + var env []string + if protocol != "" { + env = append(env, "GIT_PROTOCOL="+protocol) + } + args = append(append([]string{"upload-pack", "--stateless-rpc"}, args...), rp.dir) + cmd := c.command(r.Context(), env, args...) + cmd.Stdin = stdin + cmd.Stdout = w + stderr := &limitedBuffer{max: maxStderr} + cmd.Stderr = stderr + if err := cmd.Run(); err != nil && r.Context().Err() == nil { + c.logger.Warn("git cache: upload-pack failed", + "repository", rp.key, "error", err, "stderr", strings.TrimSpace(stderr.String())) + } +} + +func gitProtocol(r *http.Request) string { + if protocol := r.Header.Get("Git-Protocol"); gitProtocolPattern.MatchString(protocol) { + return protocol + } + return "" +} + +func isProtocolV2(protocol string) bool { + for field := range strings.SplitSeq(protocol, ":") { + if field == "version=2" { + return true + } + } + return false +} + +func setNoCache(w http.ResponseWriter) { + w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") +} diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go new file mode 100644 index 00000000..b81370e4 --- /dev/null +++ b/internal/gitcache/serve_test.go @@ -0,0 +1,196 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "os/exec" + "slices" + "strings" + "sync" + "testing" +) + +// TestRequestBodies sends the bodies git sends for large requests: gzipped, +// or chunked once one outgrows http.postBuffer. +func TestRequestBodies(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + lsRefs := pktLine("command=ls-refs\n") + pktDelim + pktLine("ref-prefix refs/heads/\n") + pktFlush + var gzipped bytes.Buffer + gz := gzip.NewWriter(&gzipped) + _, _ = io.WriteString(gz, lsRefs) + _ = gz.Close() + for _, tt := range []struct { + name string + body io.Reader + encoding string + }{ + // A reader of unknown length is sent chunked. + {"chunked", io.MultiReader(strings.NewReader(lsRefs)), ""}, + {"gzip", &gzipped, "gzip"}, + } { + req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + if tt.encoding != "" { + req.Header.Set("Content-Encoding", tt.encoding) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "refs/heads/main") { + t.Errorf("%s: status %d, body %q", tt.name, resp.StatusCode, out) + } + } +} + +// TestCloneWithChunkedRequests clones a repository with enough refs that +// git's requests, to the cache and from the cache to the upstream, outgrow +// http.postBuffer and are sent chunked. +func TestCloneWithChunkedRequests(t *testing.T) { + const tags = 2000 + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + var stream strings.Builder + for i := 1; i <= tags; i++ { + fmt.Fprintf(&stream, "commit refs/heads/main\nmark :%d\ncommitter test %d +0000\ndata 1\n\n", i, i) + if i == 1 { + stream.WriteString("from refs/heads/main^0\n") + } + fmt.Fprintf(&stream, "reset refs/tags/t%d\nfrom :%d\n\n", i, i) + } + importer := exec.Command("git", "fast-import", "--quiet") + importer.Dir, importer.Env, importer.Stdin = work, gitEnv(t.TempDir()), strings.NewReader(stream.String()) + if out, err := importer.CombinedOutput(); err != nil { + t.Fatalf("git fast-import: %v\n%s", err, out) + } + runGit(t, work, "push", "--quiet", "origin", "main", "--tags") + tc := newTestCache(t, upstream.server.URL, nil) + + // One want per tag is about 100KB, past a postBuffer just above git's + // minimum. + clone := tc.clone(t, "repo", "-c", "http.postBuffer=66000") + if got := strings.Count(runGit(t, clone, "tag", "--list"), "\n") + 1; got != tags+1 { + t.Errorf("cloned %d tags, want %d", got, tags+1) + } +} + +func TestParseV0Advertisement(t *testing.T) { + oid := strings.Repeat("a", 40) + peeled := strings.Repeat("b", 40) + adv := pktLine(oid+" HEAD\x00multi_ack symref=HEAD:refs/heads/trunk agent=git/2\n") + + pktLine(oid+" refs/heads/trunk\n") + + pktLine(oid+" refs/tags/v1\n") + + pktLine(peeled+" refs/tags/v1^{}\n") + + pktLine(oid+" refs/changes/01/1/1\n") + + pktLine(oid+" refs/pull/1/head\n") + + pktFlush + pkts := newPktReader(strings.NewReader(adv)) + kind, line, err := pkts.next() + if err != nil { + t.Fatal(err) + } + remote, err := parseV0Advertisement(pkts, kind, line) + if err != nil { + t.Fatal(err) + } + if remote.head != "refs/heads/trunk" { + t.Errorf("head = %q", remote.head) + } + want := map[string]string{"refs/heads/trunk": oid, "refs/tags/v1": oid} + if len(remote.refs) != len(want) || remote.refs["refs/heads/trunk"] != oid || remote.refs["refs/tags/v1"] != oid { + t.Errorf("refs = %v, want %v", remote.refs, want) + } +} + +// TestCommitOnNoBranchIsFetchedByID does what actions/checkout does: fetch +// one commit by object ID, here a pull request head that is on no branch. +func TestCommitOnNoBranchIsFetchedByID(t *testing.T) { + for _, version := range []string{"0", "2"} { + t.Run("protocol v"+version, func(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + runGit(t, work, "checkout", "--quiet", "-b", "pr") + sha := upstream.commit(work, "pull request") + runGit(t, work, "push", "--quiet", "origin", "pr:refs/pull/1/head") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + sessions := upstream.gitSessions.Load() + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if out, err := tryGit(t, dir, "-c", "protocol.version="+version, "fetch", "--quiet", "--depth=1", tc.url("repo"), sha); err != nil { + t.Errorf("fetch by ID: %v\n%s", err, out) + return + } + if got := runGit(t, dir, "rev-parse", "FETCH_HEAD"); got != sha { + t.Errorf("FETCH_HEAD = %s, want %s", got, sha) + } + }) + } + wg.Wait() + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches for the commit, want 1", got) + } + if refs := runGit(t, "", "ls-remote", tc.url("repo")); strings.Contains(refs, wantedPrefix) { + t.Errorf("advertised %s refs:\n%s", wantedPrefix, refs) + } + + // A refresh that prunes branches keeps the commit. + runGit(t, work, "push", "--quiet", "origin", "main:refs/heads/new") + tc.clone(t, "repo") + if refs := tc.cachedRefs(t); !strings.Contains(refs, wantedPrefix) || !strings.Contains(refs, "/"+sha) { + t.Errorf("refresh dropped the fetched commit:\n%s", refs) + } + }) + } +} + +func TestReadWants(t *testing.T) { + a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) + tests := map[string]struct { + body string + want []string + }{ + "v0": { + pktLine("want "+a+" multi_ack side-band-64k\n") + pktLine("want "+b+"\n") + pktLine("deepen 1\n") + pktFlush + pktLine("have "+a+"\n") + pktFlush, + []string{a, b}, + }, + "v2 fetch": { + pktLine("command=fetch\n") + pktLine("agent=git/2\n") + pktDelim + pktLine("thin-pack\n") + pktLine("want "+a+"\n") + pktLine("have "+b+"\n") + pktLine("want "+b+"\n") + pktLine("done\n") + pktFlush, + []string{a, b}, + }, + "v2 ls-refs": { + pktLine("command=ls-refs\n") + pktDelim + pktLine("want "+a+"\n") + pktFlush, + nil, + }, + "not an object ID": { + pktLine("want HEAD\n") + pktFlush, + nil, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + wants, replay := readWants(strings.NewReader(tt.body)) + if !slices.Equal(wants, tt.want) { + t.Errorf("wants = %v, want %v", wants, tt.want) + } + if got, _ := io.ReadAll(replay); string(got) != tt.body { + t.Errorf("replayed %q, want the whole body", got) + } + }) + } +} diff --git a/internal/gitcache/upstream.go b/internal/gitcache/upstream.go new file mode 100644 index 00000000..a21f39df --- /dev/null +++ b/internal/gitcache/upstream.go @@ -0,0 +1,210 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + advertisementType = "application/x-git-upload-pack-advertisement" + requestType = "application/x-git-upload-pack-request" + resultType = "application/x-git-upload-pack-result" + // Some servers only speak the smart protocol to clients that look like git. + gitUserAgent = "git/2 (git-pkgs-proxy)" + // maxRefsResponse bounds a ref listing; branches and tags of even large + // repositories take a few megabytes. + maxRefsResponse = 64 << 20 + + headsPrefix = "refs/heads/" + tagsPrefix = "refs/tags/" +) + +var ( + // ErrNotFound is returned when the upstream has no such repository. + ErrNotFound = errors.New("repository not found upstream") + // ErrRefused covers authentication failures and redirects. The cached + // copy is not served after one, since the upstream may have withdrawn + // access to it. + ErrRefused = errors.New("upstream refused the request") + // ErrTooLarge is returned for a repository larger than git.max_repo_size. + ErrTooLarge = errors.New("repository is larger than git.max_repo_size") +) + +// remoteRefs is what a repository advertises: its branch and tag tips, and +// the branch HEAD points at. +type remoteRefs struct { + refs map[string]string + head string +} + +// listUpstreamRefs asks the upstream for its branches, tags and HEAD. With +// protocol v2 the server filters by prefix, which matters for Gerrit, where +// every patch set is a ref. Servers without v2 send every ref, and the rest +// are dropped here. +func (c *Cache) listUpstreamRefs(ctx context.Context, repoURL string) (remoteRefs, error) { + resp, err := c.upstreamRequest(ctx, http.MethodGet, repoURL+"/info/refs?service=git-upload-pack", "") + if err != nil { + return remoteRefs{}, err + } + defer func() { _ = resp.Body.Close() }() + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, advertisementType) { + return remoteRefs{}, fmt.Errorf("upstream did not answer as a smart HTTP Git server (content type %q)", ct) + } + + pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) + kind, line, err := pkts.next() + if err == nil && kind == pktData && strings.HasPrefix(line, "# service=") { + if kind, _, err = pkts.next(); err == nil && kind != pktFlushKind { + err = errors.New("no flush after the service line") + } + if err == nil { + kind, line, err = pkts.next() + } + } + if err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + + if kind == pktData && line == "version 2" { + return c.lsRefs(ctx, repoURL) + } + if kind == pktData && line == "version 1" { + if kind, line, err = pkts.next(); err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + } + return parseV0Advertisement(pkts, kind, line) +} + +// lsRefs runs the protocol v2 ls-refs command. +func (c *Cache) lsRefs(ctx context.Context, repoURL string) (remoteRefs, error) { + body := pktLine("command=ls-refs\n") + pktDelim + + pktLine("symrefs\n") + + pktLine("ref-prefix HEAD\n") + + pktLine("ref-prefix "+headsPrefix+"\n") + + pktLine("ref-prefix "+tagsPrefix+"\n") + + pktFlush + resp, err := c.upstreamRequest(ctx, http.MethodPost, repoURL+"/git-upload-pack", body) + if err != nil { + return remoteRefs{}, err + } + defer func() { _ = resp.Body.Close() }() + + remote := remoteRefs{refs: make(map[string]string)} + pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) + for { + kind, line, err := pkts.next() + if err != nil { + return remoteRefs{}, fmt.Errorf("reading ls-refs response: %w", err) + } + if kind == pktFlushKind { + return remote, nil + } + if kind != pktData { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 2 { //nolint:mnd // object id and ref name + return remoteRefs{}, fmt.Errorf("invalid ls-refs line %q", line) + } + if fields[1] == "HEAD" { + for _, attr := range fields[2:] { + if target, ok := strings.CutPrefix(attr, "symref-target:"); ok { + remote.head = target + } + } + continue + } + remote.add(fields[1], fields[0]) + } +} + +// parseV0Advertisement reads a protocol v0 ref advertisement whose first +// packet has already been read. The first ref carries the capabilities, +// including where HEAD points. +func parseV0Advertisement(pkts *pktReader, kind pktKind, line string) (remoteRefs, error) { + remote := remoteRefs{refs: make(map[string]string)} + first := true + for kind != pktFlushKind { + if kind == pktData { + ref, caps, _ := strings.Cut(line, "\x00") + if first { + remote.head = symrefTarget(caps, "HEAD") + first = false + } + oid, name, ok := strings.Cut(ref, " ") + if !ok { + return remoteRefs{}, fmt.Errorf("invalid ref advertisement line %q", ref) + } + remote.add(name, oid) + } + var err error + if kind, line, err = pkts.next(); err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + } + return remote, nil +} + +// add keeps branches and tags, leaving out peeled tag entries. +func (r remoteRefs) add(name, oid string) { + if (strings.HasPrefix(name, headsPrefix) || strings.HasPrefix(name, tagsPrefix)) && !strings.HasSuffix(name, "^{}") { + r.refs[name] = oid + } +} + +func symrefTarget(caps, ref string) string { + for _, capability := range strings.Fields(caps) { + if value, ok := strings.CutPrefix(capability, "symref="+ref+":"); ok { + return value + } + } + return "" +} + +// upstreamRequest sends a smart HTTP request through the upstream client and +// classifies refusals. A non-empty body is sent as an upload-pack request. +func (c *Cache) upstreamRequest(ctx context.Context, method, target, body string) (*http.Response, error) { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) + if err != nil { + return nil, err + } + req.Header.Set("Git-Protocol", "version=2") + req.Header.Set("User-Agent", gitUserAgent) + if body != "" { + req.Header.Set("Content-Type", requestType) + req.Header.Set("Accept", resultType) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("upstream request: %w", err) + } + if err := upstreamStatusError(resp.StatusCode); err != nil { + _ = resp.Body.Close() + return nil, err + } + return resp, nil +} + +func upstreamStatusError(status int) error { + switch { + case status == http.StatusOK: + return nil + case status == http.StatusNotFound || status == http.StatusGone: + return fmt.Errorf("%w: status %d", ErrNotFound, status) + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return fmt.Errorf("%w: status %d", ErrRefused, status) + case status >= http.StatusMultipleChoices && status < http.StatusBadRequest: + return fmt.Errorf("%w: redirect (status %d) not followed", ErrRefused, status) + default: + return fmt.Errorf("upstream returned status %d", status) + } +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go new file mode 100644 index 00000000..eec8fc45 --- /dev/null +++ b/internal/gitcache/wants.go @@ -0,0 +1,264 @@ +package gitcache + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "slices" + "strconv" + "strings" + "time" +) + +const ( + // wantedPrefix holds commits fetched because a client asked for them by + // object ID, such as a pull request head that is on no branch, as + // {wantedPrefix}{unix time}/{object ID}. The refs keep them from being + // garbage collected, and are hidden from clients. + wantedPrefix = "refs/proxy/wanted/" + // wantedMaxAge is how long a commit fetched by ID is kept while no + // cached branch or tag includes it. + wantedMaxAge = 7 * 24 * time.Hour + // wantRetryAfter is how long a want the upstream could not provide is + // not asked for again. Short, since a commit can reach a CI job before + // it reaches every upstream replica. A want too large for what + // max_repo_size leaves waits tooLargeRetryAfter instead. + wantRetryAfter = time.Minute + // maxFailedWants bounds the record of failed wants. + maxFailedWants = 4096 + // maxWantScan bounds how much of a request body is read for its wants. + maxWantScan = 1 << 20 + // maxWantFetch bounds the missing objects one request may have fetched. + maxWantFetch = 32 + // maxWantSplit bounds the wants of a failed fetch that are fetched again + // one at a time; a larger batch is refused as a whole. + maxWantSplit = 4 +) + +// readWants reads the object IDs a fetch request wants, from its first +// section, and returns them with a reader that replays the whole body. In +// protocol v0 that section holds the wants; in v2 it is the whole request. +func readWants(body io.Reader) ([]string, io.Reader) { + var consumed bytes.Buffer + pkts := newPktReader(io.TeeReader(io.LimitReader(body, maxWantScan), &consumed)) + var wants []string + for first := true; ; first = false { + kind, line, err := pkts.next() + if err != nil || kind == pktFlushKind { + break + } + if kind != pktData { + continue + } + if command, ok := strings.CutPrefix(line, "command="); ok && first && command != "fetch" { + break + } + if rest, ok := strings.CutPrefix(line, "want "); ok { + if oid, _, _ := strings.Cut(rest, " "); isObjectID(oid) { + wants = append(wants, oid) + } + } + } + return wants, io.MultiReader(&consumed, body) +} + +// fetchMissingWants fetches the wanted objects the repository lacks. Wants +// for branch and tag tips are already there after the refresh; this covers +// commits a client names directly, as actions/checkout does. +func (c *Cache) fetchMissingWants(ctx context.Context, rp *repo, repoURL string, wants []string) error { + missing, err := c.missingObjects(ctx, rp.dir, wants) + if err != nil { + return err + } + missing = c.withoutFailedWants(rp.key, missing) + if len(missing) == 0 || len(missing) > maxWantFetch { + return nil + } + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + // Another request may have fetched them while this one waited. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil || len(missing) == 0 { + return err + } + defer c.evict() + + err = c.fetchWants(ctx, rp, repoURL, missing) + if err == nil || ctx.Err() != nil { + return err + } + if len(missing) == 1 || len(missing) > maxWantSplit || + errors.Is(err, ErrTooLarge) || errors.Is(err, errUpstreamUnanswered) { + // Fetched one at a time, wants too large for what max_repo_size + // leaves would exhaust the same budget, and an upstream that did + // not answer would be waited on once per want. + c.recordFailedWants(rp.key, missing, err) + return err + } + // Find which wants failed, so a good one asked for with a bad one is + // not refused too. These fetches hold the repository's write lock, so + // only a small batch is split. + var errs []error + for _, oid := range missing { + if err := c.fetchWants(ctx, rp, repoURL, []string{oid}); err != nil { + if ctx.Err() != nil { + return err + } + c.recordFailedWants(rp.key, []string{oid}, err) + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// fetchWants fetches oids from the upstream under wanted refs, within what +// max_repo_size leaves. A want that does not fit is refused, not the +// repository: transfer.unpackLimit makes a cut-off transfer leave nothing +// behind. +func (c *Cache) fetchWants(ctx context.Context, rp *repo, repoURL string, oids []string) error { + limit, err := c.fetchLimit(rp) + if err != nil { + return err + } + now := time.Now() + refspecs := make([]string, len(oids)) + for i, oid := range oids { + refspecs[i] = oid + ":" + wantedRef(oid, now) + } + err = c.fetch(ctx, rp.dir, repoURL, limit, []string{"--atomic"}, refspecs...) + rp.size.Store(dirSize(rp.dir)) + return err +} + +func wantedRef(oid string, at time.Time) string { + return wantedPrefix + strconv.FormatInt(at.Unix(), 10) + "/" + oid +} + +// pruneWanted deletes wanted refs that are no longer needed: those older +// than wantedMaxAge, and, after a fetch has moved branches or tags, those +// whose commit a cached branch or tag now includes. +func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fetched bool) error { + if len(wanted) == 0 { + return nil + } + cutoff := time.Now().Add(-wantedMaxAge).Unix() + var drop []string + keep := make(map[string]string, len(wanted)) + for _, ref := range wanted { + created, oid, ok := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if !ok || err != nil || unix < cutoff || !isObjectID(oid) { + drop = append(drop, ref) + continue + } + keep[oid] = ref + } + var mergedErr error + if fetched && len(keep) > 0 { + // rev-list prints what the wanted commits, read from stdin, reach + // and no branch or tag does, so a wanted commit it leaves out is + // merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") + out, err := cmd.Output() + if err != nil { + mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) + } else { + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + for oid, ref := range keep { + if !unmerged[oid] { + drop = append(drop, ref) + } + } + } + } + if len(drop) == 0 { + return mergedErr + } + var stdin strings.Builder + for _, ref := range drop { + stdin.WriteString("delete " + ref + "\n") + } + cmd := c.command(ctx, nil, "-C", dir, "update-ref", "--stdin") + cmd.Stdin = strings.NewReader(stdin.String()) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("deleting wanted refs: %w: %s", err, strings.TrimSpace(string(out))) + } + return mergedErr +} + +// withoutFailedWants drops the object IDs a recent fetch for the +// repository could not get. +func (c *Cache) withoutFailedWants(key string, oids []string) []string { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + return slices.DeleteFunc(oids, func(oid string) bool { + until, ok := c.failedWants[key+" "+oid] + return ok && now.Before(until) + }) +} + +// recordFailedWants keeps oids from being fetched again for the repository +// for wantRetryAfter, or for tooLargeRetryAfter when they did not fit in +// max_repo_size. +func (c *Cache) recordFailedWants(key string, oids []string, err error) { + now := time.Now() + until := now.Add(wantRetryAfter) + if errors.Is(err, ErrTooLarge) { + until = now.Add(tooLargeRetryAfter) + } + c.mu.Lock() + defer c.mu.Unlock() + for k, expiry := range c.failedWants { + if !now.Before(expiry) { + delete(c.failedWants, k) + } + } + for _, oid := range oids { + if len(c.failedWants) >= maxFailedWants { + return + } + c.failedWants[key+" "+oid] = until + } +} + +// missingObjects returns the object IDs in oids the repository does not have. +func (c *Cache) missingObjects(ctx context.Context, dir string, oids []string) ([]string, error) { + if len(oids) == 0 { + return nil, nil + } + cmd := c.command(ctx, nil, "-C", dir, "cat-file", "--batch-check") + cmd.Stdin = strings.NewReader(strings.Join(oids, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + var missing []string + for line := range strings.Lines(string(out)) { + if oid, ok := strings.CutSuffix(strings.TrimSuffix(line, "\n"), " missing"); ok { + missing = append(missing, oid) + } + } + return missing, nil +} + +func isObjectID(s string) bool { + if len(s) != 40 && len(s) != 64 { //nolint:mnd // SHA-1 and SHA-256 hex lengths + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} diff --git a/internal/gitcache/wants_test.go b/internal/gitcache/wants_test.go new file mode 100644 index 00000000..f01df062 --- /dev/null +++ b/internal/gitcache/wants_test.go @@ -0,0 +1,319 @@ +package gitcache + +import ( + "crypto/rand" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// pushPullRequest commits on a new branch from main, with data in a file if +// given, and pushes it only as refs/pull/{n}/head, so the commit is on no +// branch or tag upstream. +func (u *fakeUpstream) pushPullRequest(work string, n int, data []byte) string { + u.t.Helper() + branch := "pr" + strconv.Itoa(n) + runGit(u.t, work, "checkout", "--quiet", "-b", branch, "main") + if data != nil { + if err := os.WriteFile(filepath.Join(work, branch+".bin"), data, 0o600); err != nil { + u.t.Fatal(err) + } + runGit(u.t, work, "add", branch+".bin") + } + runGit(u.t, work, "commit", "--quiet", "--allow-empty", "-m", branch) + runGit(u.t, work, "push", "--quiet", "origin", "HEAD:refs/pull/"+strconv.Itoa(n)+"/head") + runGit(u.t, work, "checkout", "--quiet", "main") + return runGit(u.t, work, "rev-parse", branch) +} + +// fetchByID fetches one commit by ID from url, as actions/checkout does. +func fetchByID(t *testing.T, url, sha string) error { + t.Helper() + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + _, err := tryGit(t, dir, "fetch", "--quiet", "--depth=1", url, sha) + return err +} + +func randomBytes(n int) []byte { + b := make([]byte, n) + _, _ = rand.Read(b) + return b +} + +func (tc *testCache) wantedRefs(t *testing.T) []string { + t.Helper() + var refs []string + for line := range strings.Lines(tc.cachedRefs(t)) { + if _, name, _ := strings.Cut(strings.TrimSpace(line), " "); strings.HasPrefix(name, wantedPrefix) { + refs = append(refs, name) + } + } + return refs +} + +func TestMergedWantedCommitIsUnpinned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Fatalf("wanted refs %v, want one", refs) + } + + runGit(t, work, "push", "--quiet", "origin", "pr1:main") + tc.clone(t, "repo") + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v after the commit was merged, want none", refs) + } +} + +func TestOldWantedCommitIsUnpinned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + old := upstream.pushPullRequest(work, 1, nil) + recent := upstream.pushPullRequest(work, 2, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + for _, sha := range []string{old, recent} { + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + } + + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + if strings.HasSuffix(ref, "/"+old) { + runGit(t, cached, "update-ref", "-d", ref) + } + } + runGit(t, cached, "update-ref", wantedRef(old, time.Now().Add(-wantedMaxAge-time.Hour)), old) + + tc.clone(t, "repo") + refs := tc.wantedRefs(t) + if len(refs) != 1 || !strings.HasSuffix(refs[0], "/"+recent) { + t.Errorf("wanted refs %v, want only the recent commit's", refs) + } +} + +// TestRepositoryTooLargeToRefreshIsClonedAgain fills a repository with a +// commit fetched by ID, then grows its branch upstream past what is left of +// max_repo_size. Rather than stop following the upstream, the repository is +// dropped once idle and cloned again with only its branches and tags, which +// fit. +func TestRepositoryTooLargeToRefreshIsClonedAgain(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + tc.clone(t, "repo") + if err := fetchByID(t, tc.url("repo"), pinned); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + head := upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + // This clone's refresh does not fit. It is served the cached copy, or, + // if the repository is dropped between its requests, the new clone. + tc.clone(t, "repo") + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s once cloned again", got, head) + } + waitIdle(t, tc.Cache) + if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", pinned); err == nil { + t.Error("the repository still holds the commit fetched by ID, so it was not cloned again") + } +} + +// TestTooLargeWantKeepsRepository asks for a commit by ID that does not fit +// in what max_repo_size leaves. The commit is refused, for a while, and the +// repository is kept, with nothing of the cut-off transfer left in it. +func TestTooLargeWantKeepsRepository(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + large := upstream.pushPullRequest(work, 1, randomBytes(200<<10)) + small := upstream.pushPullRequest(work, 2, nil) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 100<<10 }) + tc.clone(t, "repo") + + if err := fetchByID(t, tc.url("repo"), large); err == nil { + t.Fatal("fetched a commit larger than max_repo_size leaves") + } + waitIdle(t, tc.Cache) + cached := filepath.Join(tc.dir, "up", "repo.git") + if !fileExists(cached) { + t.Fatal("the repository was dropped") + } + if _, err := tryGit(t, cached, "cat-file", "-e", large); err == nil { + t.Error("the cut-off transfer left the commit behind") + } + if leftovers, _ := filepath.Glob(filepath.Join(cached, "objects", "pack", "tmp_*")); len(leftovers) != 0 { + t.Errorf("the cut-off transfer left %v", leftovers) + } + tc.mu.Lock() + until := tc.failedWants["up/repo.git "+large] + tc.mu.Unlock() + if time.Until(until) <= wantRetryAfter { + t.Errorf("the large commit is refused until %s, want for tooLargeRetryAfter", until) + } + + before := upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), large); err == nil { + t.Fatal("fetched the large commit on a second try") + } + if got := upstream.gitRequests.Load() - before; got != 0 { + t.Errorf("the second try made %d upstream fetch requests, want none", got) + } + if err := fetchByID(t, tc.url("repo"), small); err != nil { + t.Errorf("fetch of another commit by ID: %v", err) + } +} + +// TestWantsFetchedTogetherFailSeparately asks for a good and a bad commit +// in one request. The good one is still fetched, and only the bad one is +// refused for a while. +func TestWantsFetchedTogetherFailSeparately(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + good := upstream.pushPullRequest(work, 1, nil) + bad := hex.EncodeToString(randomBytes(20)) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), good, bad); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + waitIdle(t, tc.Cache) + if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", good); err != nil { + t.Error("the good commit was not fetched") + } + tc.mu.Lock() + _, goodRefused := tc.failedWants["up/repo.git "+good] + _, badRefused := tc.failedWants["up/repo.git "+bad] + tc.mu.Unlock() + if goodRefused || !badRefused { + t.Errorf("refused good=%v bad=%v, want only the bad commit", goodRefused, badRefused) + } +} + +func TestFailedWantIsNotRetriedAtOnce(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + missing := hex.EncodeToString(randomBytes(20)) + + before := upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), missing); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + if upstream.gitRequests.Load() == before { + t.Fatal("the first request did not ask the upstream") + } + before = upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), missing); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + if got := upstream.gitRequests.Load() - before; got != 0 { + t.Errorf("the second request made %d upstream fetch requests, want none", got) + } +} + +func TestWantsAreNotFetchedWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + server := httptest.NewServer(testFront(tc.Cache, upstream.server.URL, false)) + defer server.Close() + url := server.URL + "/git/up/repo" + runGit(t, "", "clone", "--quiet", url, filepath.Join(t.TempDir(), "clone")) + + sessions := upstream.gitSessions.Load() + if err := fetchByID(t, url, sha); err == nil { + t.Error("fetched a commit by ID with FetchWants off") + } + if got := upstream.gitSessions.Load() - sessions; got != 0 { + t.Errorf("upstream saw %d fetches, want none", got) + } +} + +// TestWantsDuringAnOutageAreNotFetchedOneByOne asks for two commits while +// the upstream is down. One failed fetch records both, rather than a fetch +// per commit, each waiting on the upstream. +func TestWantsDuringAnOutageAreNotFetchedOneByOne(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + first := upstream.pushPullRequest(work, 1, nil) + second := upstream.pushPullRequest(work, 2, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusServiceUnavailable) + sessions := upstream.gitSessions.Load() + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), first, second); err == nil { + t.Fatal("fetched commits while the upstream was down") + } + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } + tc.mu.Lock() + _, firstRefused := tc.failedWants["up/repo.git "+first] + _, secondRefused := tc.failedWants["up/repo.git "+second] + tc.mu.Unlock() + if !firstRefused || !secondRefused { + t.Errorf("refused first=%v second=%v, want both", firstRefused, secondRefused) + } +} + +// TestLargeBatchOfFailedWantsIsNotSplit asks for more unknown commits at +// once than are fetched again one by one, which would hold the repository's +// write lock for a fetch per commit. +func TestLargeBatchOfFailedWantsIsNotSplit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + args := []string{"fetch", "--quiet", tc.url("repo")} + for range maxWantSplit + 1 { + args = append(args, hex.EncodeToString(randomBytes(20))) + } + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + sessions := upstream.gitSessions.Load() + if _, err := tryGit(t, dir, args...); err == nil { + t.Fatal("fetched commits the upstream does not have") + } + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } +} diff --git a/internal/handler/git.go b/internal/handler/git.go new file mode 100644 index 00000000..4af07ea0 --- /dev/null +++ b/internal/handler/git.go @@ -0,0 +1,213 @@ +package handler + +import ( + "errors" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + gitEcosystem = "git" + + gitInfoRefs = "/info/refs" + gitUploadPack = "/git-upload-pack" + gitReceivePack = "/git-receive-pack" + gitRequestType = "application/x-git-upload-pack-request" + + // gitRequestTimeout replaces the server's read and write timeouts for Git + // requests: a cold clone of a large repository keeps the client waiting + // before the first byte, a large pack takes a while to send, and the rest + // of a request body is read only after any fetch it needs. + gitRequestTimeout = 2 * time.Hour +) + +// GitHandler serves Git repositories read-only over smart HTTP at +// /git/{name}/{repo}, from a local cache of the upstream.git hosts. Clients +// clone and fetch with ordinary git commands, usually through an insteadOf +// rewrite. Pushing and the dumb protocol are refused. +type GitHandler struct { + proxy *Proxy + cache *gitcache.Cache + upstreams map[string]gitUpstream +} + +type gitUpstream struct { + config.GitUpstream + // dir names the upstream's directory in the cache (gitcache.UpstreamDir). + dir string + // fetchWants is off when upstream.auth has credentials for the host. A + // fetch by ID there could reach commits outside the repos allowlist, + // as GitHub serves any commit in a repository's fork network. + fetchWants bool +} + +// NewGitHandler creates a Git handler serving upstream.git from cache. +func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler { + h := &GitHandler{proxy: proxy, cache: cache, upstreams: make(map[string]gitUpstream, len(upstream.Git))} + for name, git := range upstream.Git { + git.URL = strings.TrimSuffix(git.URL, "/") + h.upstreams[name] = gitUpstream{ + GitUpstream: git, + dir: gitcache.UpstreamDir(name, git.URL), + fetchWants: !upstream.HasAuthWithin(git.URL), + } + } + return h +} + +// Routes returns the HTTP handler for Git requests. Mount this at /git. +func (h *GitHandler) Routes() http.Handler { + return http.HandlerFunc(h.serve) +} + +// gitRequest is a parsed /git/{name}/{repo}/{endpoint} request. +type gitRequest struct { + infoRefs bool + fetchWants bool + // key names the repository in the cache: {name}-{URL hash}/{repo}.git, + // with any .git the client gave removed from repo first, so both + // spellings share one copy. + key string + // repoURL is the upstream repository, spelled as the client spelled it. + repoURL string +} + +func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { + req, status, msg := h.parse(r) + if status != 0 { + http.Error(w, msg, status) + return + } + deadline := time.Now().Add(gitRequestTimeout) + rc := http.NewResponseController(w) + if err := errors.Join(rc.SetReadDeadline(deadline), rc.SetWriteDeadline(deadline)); err != nil { + h.proxy.Logger.Warn("git: cannot extend the request deadlines", "error", err) + } + // upload-pack writes its response while its stdin is still attached to + // the request body. Without full duplex, an HTTP/1 server reads off and + // closes whatever body is left once the response is flushed. + if !req.infoRefs { + if err := rc.EnableFullDuplex(); err != nil { + h.proxy.Logger.Warn("git: cannot enable full duplex", "error", err) + } + } + + lease, err := h.cache.Open(r.Context(), req.key, req.repoURL, + gitcache.OpenOptions{Refresh: req.infoRefs, FetchWants: req.fetchWants}) + if err != nil { + h.serveError(w, r, err) + return + } + defer lease.Release() + if !req.infoRefs { + lease.ServeUploadPack(w, r) + return + } + if lease.Hit() { + metrics.RecordArtifactCacheHit(gitEcosystem) + } else { + metrics.RecordArtifactCacheMiss(gitEcosystem) + } + lease.ServeInfoRefs(w, r) +} + +func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { + path := strings.TrimPrefix(r.URL.Path, "/") + var req gitRequest + switch { + case strings.HasSuffix(path, gitInfoRefs): + path = strings.TrimSuffix(path, gitInfoRefs) + req.infoRefs = true + switch r.URL.Query().Get("service") { + case "git-upload-pack": + case "git-receive-pack": + return req, http.StatusForbidden, "pushing is not supported" + default: + return req, http.StatusForbidden, "only the smart HTTP protocol is supported" + } + if r.Method != http.MethodGet { + return req, http.StatusMethodNotAllowed, "method not allowed" + } + case strings.HasSuffix(path, gitUploadPack): + path = strings.TrimSuffix(path, gitUploadPack) + if r.Method != http.MethodPost { + return req, http.StatusMethodNotAllowed, "method not allowed" + } + if r.Header.Get("Content-Type") != gitRequestType { + return req, http.StatusUnsupportedMediaType, "unsupported content type" + } + case strings.HasSuffix(path, gitReceivePack): + return req, http.StatusForbidden, "pushing is not supported" + default: + return req, http.StatusNotFound, "not found" + } + + name, repoPath, _ := strings.Cut(path, "/") + upstream, ok := h.upstreams[name] + if !ok { + return req, http.StatusNotFound, "unknown upstream" + } + segments := strings.Split(repoPath, "/") + last := len(segments) - 1 + suffix := "" + if trimmed, ok := strings.CutSuffix(segments[last], ".git"); ok { + segments[last], suffix = trimmed, ".git" + } + escaped := make([]string, len(segments)) + for i, segment := range segments { + if !validGitSegment(segment, i == last) { + return req, http.StatusNotFound, "invalid repository path" + } + escaped[i] = url.PathEscape(segment) + } + repo := strings.Join(segments, "/") + if !upstream.AllowsRepo(repo) { + // As for a missing repository, so a prober learns nothing about + // the allowlist. + return req, http.StatusNotFound, "repository not found" + } + req.key = upstream.dir + "/" + repo + ".git" + req.repoURL = upstream.URL + "/" + strings.Join(escaped, "/") + suffix + req.fetchWants = upstream.fetchWants + return req, 0, "" +} + +// validGitSegment checks one segment of a repository path, which becomes both +// part of a directory under the cache and part of the upstream URL. Only the +// last segment may end in .git, so one repository's directory never sits +// inside another's. +func validGitSegment(segment string, last bool) bool { + if segment == "" || segment == "." || segment == ".." || (!last && strings.HasSuffix(segment, ".git")) { + return false + } + for _, r := range segment { + if r < ' ' || r == 0x7f || r == '\\' { + return false + } + } + return true +} + +// serveError answers a request whose repository could not be opened. The +// cache has already logged why. +func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { + if r.Context().Err() != nil { + return + } + switch { + case errors.Is(err, gitcache.ErrNotFound): + http.Error(w, "repository not found", http.StatusNotFound) + case errors.Is(err, gitcache.ErrRefused): + http.Error(w, "upstream refused the request", http.StatusBadGateway) + case errors.Is(err, gitcache.ErrTooLarge): + http.Error(w, "repository is larger than git.max_repo_size", http.StatusInsufficientStorage) + default: + http.Error(w, "upstream unavailable", http.StatusBadGateway) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go new file mode 100644 index 00000000..ec0b6306 --- /dev/null +++ b/internal/handler/git_test.go @@ -0,0 +1,247 @@ +package handler + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" +) + +func TestParseGitRequest(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "up": {URL: "https://git.example/r/"}, + "limited": {URL: "https://git.example", Repos: []string{"allowed/*"}}, + }}) + up, limited := h.upstreams["up"].dir, h.upstreams["limited"].dir + tests := []struct { + method, target string + status int + key, repoURL string + }{ + {"GET", "/up/repo.git/info/refs?service=git-upload-pack", 0, up + "/repo.git", "https://git.example/r/repo.git"}, + {"GET", "/up/a/b/info/refs?service=git-upload-pack", 0, up + "/a/b.git", "https://git.example/r/a/b"}, + {"POST", "/up/a/b%20c/git-upload-pack", 0, up + "/a/b c.git", "https://git.example/r/a/b%20c"}, + {"GET", "/up/a.git/b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/a/../b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/a//b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/.git/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/repo/info/refs?service=git-receive-pack", http.StatusForbidden, "", ""}, + {"POST", "/up/repo/git-receive-pack", http.StatusForbidden, "", ""}, + {"GET", "/up/repo/info/refs", http.StatusForbidden, "", ""}, + {"GET", "/up/repo/objects/info/packs", http.StatusNotFound, "", ""}, + {"POST", "/up/repo/info/refs?service=git-upload-pack", http.StatusMethodNotAllowed, "", ""}, + {"GET", "/up/repo/git-upload-pack", http.StatusMethodNotAllowed, "", ""}, + {"GET", "/limited/allowed/repo/info/refs?service=git-upload-pack", 0, limited + "/allowed/repo.git", "https://git.example/allowed/repo"}, + {"GET", "/limited/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + } + for _, tt := range tests { + t.Run(tt.method+" "+tt.target, func(t *testing.T) { + r := httptest.NewRequest(tt.method, tt.target, nil) + r.Header.Set("Content-Type", gitRequestType) + req, status, _ := h.parse(r) + if status != tt.status { + t.Fatalf("status %d, want %d", status, tt.status) + } + if status == 0 && (req.key != tt.key || req.repoURL != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", req.key, req.repoURL, tt.key, tt.repoURL) + } + }) + } + + r := httptest.NewRequest(http.MethodPost, "/up/repo/git-upload-pack", nil) + if _, status, _ := h.parse(r); status != http.StatusUnsupportedMediaType { + t.Errorf("POST without the request content type: status %d, want 415", status) + } +} + +// TestGitUpstreamCacheDirectory checks the cache directory follows the +// upstream's URL, so repointing an upstream does not serve the old host's +// repositories. +func TestGitUpstreamCacheDirectory(t *testing.T) { + dir := func(url string) string { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: url}}}) + return h.upstreams["up"].dir + } + if a, b := dir("https://git.example"), dir("https://other.example"); a == b || !strings.HasPrefix(a, "up-") { + t.Errorf("directories %q and %q, want distinct ones named after the upstream", a, b) + } + if a, b := dir("https://git.example"), dir("https://git.example/"); a != b { + t.Errorf("a trailing slash changed the directory: %q, %q", a, b) + } +} + +// TestGitFetchByIDOffWithCredentials checks commits are not fetched by ID +// for an upstream that upstream.auth has credentials for: there a fetch by +// ID could reach repositories outside its repos allowlist. +func TestGitFetchByIDOffWithCredentials(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{ + Git: map[string]config.GitUpstream{ + "public": {URL: "https://github.com"}, + "private": {URL: "https://git.example", Repos: []string{"team/*"}}, + "scoped": {URL: "https://other.example", Repos: []string{"team/*"}}, + }, + Auth: map[string]config.AuthConfig{ + "https://git.example": {Type: "bearer", Token: "t"}, + "https://other.example/team": {Type: "bearer", Token: "t"}, + }, + }) + for name, want := range map[string]bool{"public": true, "private": false, "scoped": false} { + if got := h.upstreams[name].fetchWants; got != want { + t.Errorf("%s: fetchWants = %v, want %v", name, got, want) + } + } +} + +func runGitCommand(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// gitTestSetup is a GitHandler in front of a cache of one upstream +// repository, "repo", served by git http-backend. +type gitTestSetup struct { + work string + server *httptest.Server + logs *syncLog + // slow delays every upstream request by half a second. + slow atomic.Bool +} + +func newGitTestSetup(t *testing.T, readTimeout time.Duration) *gitTestSetup { + t.Helper() + execPath := runGitCommand(t, "", "--exec-path") + backendPath := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(execPath, name); fileExists(path) { + backendPath = path + } + } + if backendPath == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "repo.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") + g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + runGitCommand(t, "", "clone", "--quiet", bare, g.work) + runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + + backend := &cgi.Handler{ + Path: backendPath, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if g.slow.Load() { + time.Sleep(500 * time.Millisecond) + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(upstream.Close) + + logger := slog.New(slog.NewTextHandler(g.logs, nil)) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: t.TempDir(), + RefreshWait: 5 * time.Second, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + Logger: logger, + Upstreams: map[string]string{"up": gitcache.UpstreamDir("up", upstream.URL)}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + h := NewGitHandler(&Proxy{Logger: logger}, cache, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: upstream.URL}}}) + g.server = httptest.NewUnstartedServer(http.StripPrefix("/git", h.Routes())) + g.server.Config.ReadTimeout = readTimeout + g.server.Start() + t.Cleanup(g.server.Close) + return g +} + +func (g *gitTestSetup) url() string { + return g.server.URL + "/git/up/repo" +} + +type syncLog struct { + mu sync.Mutex + buf strings.Builder +} + +func (l *syncLog) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.Write(p) +} + +func (l *syncLog) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.String() +} + +// TestGitRequestBodyOutlastsServerReadTimeout sends a protocol v0 request +// whose haves come after a want the cache must fetch first, so upload-pack +// reads them after the server's read timeout has passed. +func TestGitRequestBodyOutlastsServerReadTimeout(t *testing.T) { + g := newGitTestSetup(t, 200*time.Millisecond) + runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "pull request") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:refs/pull/1/head") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) + + var body strings.Builder + body.WriteString(pktLine("want "+sha+" side-band-64k\n") + "0000") + for i := range 2000 { + body.WriteString(pktLine(fmt.Sprintf("have %040x\n", i+1))) + } + body.WriteString(pktLine("done\n")) + + g.slow.Store(true) + resp, err := http.Post(g.url()+"/git-upload-pack", gitRequestType, strings.NewReader(body.String())) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + out, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "PACK") { + t.Errorf("status %d, no pack in %d bytes: %.200q", resp.StatusCode, len(out), out) + } +} + +func pktLine(payload string) string { + return fmt.Sprintf("%04x%s", len(payload)+4, payload) +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/server/git_test.go b/internal/server/git_test.go new file mode 100644 index 00000000..d52f1a95 --- /dev/null +++ b/internal/server/git_test.go @@ -0,0 +1,160 @@ +package server + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func gitTestEnv(home string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } +} + +func gitCommand(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitTestEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// gitHTTPBackend finds git http-backend, which serves the test upstream, and +// skips the test where git is installed without it. +func gitHTTPBackend(t *testing.T) string { + t.Helper() + dir := gitCommand(t, "", "--exec-path") + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if _, err := os.Stat(filepath.Join(dir, name)); err == nil { + return filepath.Join(dir, name) + } + } + t.Skip("git http-backend is not installed") + return "" +} + +// syncBuffer is a log destination the server and the test share. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole +// server, with credentials from upstream.auth that only the proxy holds. +func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + gitCommand(t, "", "clone", "--quiet", bare, work) + if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte("hello\n"), 0o600); err != nil { + t.Fatal(err) + } + gitCommand(t, work, "add", "file.txt") + gitCommand(t, work, "commit", "--quiet", "-m", "first") + gitCommand(t, work, "push", "--quiet", "origin", "HEAD:main") + head := gitCommand(t, work, "rev-parse", "HEAD") + + backend := &cgi.Handler{ + Path: gitHTTPBackend(t), + Env: append(gitTestEnv(t.TempDir()), "GIT_PROJECT_ROOT="+root, "GIT_HTTP_EXPORT_ALL=1"), + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer git-secret" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + defer upstream.Close() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Upstream.AllowLoopback = true + cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} + cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} + cfg.Git.CacheDir = filepath.Join(tempDir, "git") + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + var logs syncBuffer + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(&logs, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := proxyServer.Shutdown(ctx); err != nil { + t.Errorf("shutting down server: %v", err) + } + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) + } + }() + + clone := filepath.Join(t.TempDir(), "clone") + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/private/team/repo.git", clone) + if got := gitCommand(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s", got, head) + } + if strings.Contains(logs.String(), "request deadlines") { + t.Errorf("request deadlines not extended:\n%s", logs.String()) + } + + resp, err := http.Get(cfg.BaseURL + "/git/private/other/repo.git/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("repository outside repos: status %d, want 404", resp.StatusCode) + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 0de0d1bc..0de0dba5 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -107,7 +107,7 @@ func requestEcosystem(path string) string { switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", "conan", "chef", "conda", "cran", "julia", "debian", "rpm", "arch", - "helm", "homebrew", "generic", "swift": + "helm", "homebrew", "generic", "swift", "git": return segment case "apk": return "alpine" diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 7c9eab92..b9c67c79 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -314,7 +314,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/conan/x": "conan", "/chef/x": "chef", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", - "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", + "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", "/git/x": "git", } for path, want := range mounts { if got := requestEcosystem(path); got != want { diff --git a/internal/server/server.go b/internal/server/server.go index 36ece1c5..2ed57101 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -74,6 +74,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/git-pkgs/proxy/internal/gitcache" "github.com/git-pkgs/proxy/internal/handler" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" @@ -122,6 +123,7 @@ type Server struct { metaStats metadataStatsCache sources sourceTracker scheduler *mirror.Scheduler + git *gitcache.Cache // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -270,6 +272,27 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts + if len(s.cfg.Upstream.Git) > 0 { + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: artifactClient, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + } + // Create router with Chi r := chi.NewRouter() @@ -496,6 +519,10 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) + if s.git != nil { + gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) + } } // configureScanning builds the scanner group from cfg and wires it into @@ -682,6 +709,12 @@ func (s *Server) Shutdown(ctx context.Context) error { s.waitForMirrorSchedules(ctx) + if s.git != nil { + if err := s.git.Close(); err != nil { + errs = append(errs, fmt.Errorf("git cache close: %w", err)) + } + } + if s.storage != nil { if err := s.storage.Close(); err != nil { errs = append(errs, fmt.Errorf("storage close: %w", err)) From 696fa0b219764f3d6d9eda38cac21319b7390bdf Mon Sep 17 00:00:00 2001 From: montehurd Date: Thu, 15 Oct 2026 10:28:19 -0700 Subject: [PATCH 2/9] Document setting the Git rewrite through environment variables The Git section of the configuration docs now explains what insteadOf does, with the URL a clone fetches from, and shows the same rewrites set through GIT_CONFIG_COUNT, GIT_CONFIG_KEY_n and GIT_CONFIG_VALUE_n for CI. Package managers that run git for Git dependencies inherit those variables, so their fetches are cached too; npm's GitHub tarball downloads do not go through git and are not. Refs #403 --- README.md | 3 ++- docs/configuration.md | 27 ++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ca1f0db..80730b6f 100644 --- a/README.md +++ b/README.md @@ -843,7 +843,8 @@ git config --global url."http://localhost:8080/git/github/".insteadOf https://gi Each repository is cached as a bare clone and refreshed whenever its branches or tags change upstream. See [configuration](docs/configuration.md#git-repositories) -for size limits, private repositories and outage behavior. +for setting the rewrite through environment variables, size limits, private +repositories and outage behavior. ## Configuration diff --git a/docs/configuration.md b/docs/configuration.md index f755ba4c..3942e31b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -645,12 +645,37 @@ git: stale_max_age: 24h # never serve a copy not checked for this long ``` -Clients point their remote URLs at the proxy, for example with `insteadOf`: +Clients reach the proxy through Git's `insteadOf` setting, which rewrites the +start of a URL before Git connects. After ```bash git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ ``` +`git clone https://github.com/git-pkgs/purl` fetches from +`http://localhost:8080/git/github/git-pkgs/purl`, while the clone's remote +stays `https://github.com/git-pkgs/purl`. + +In CI, the same setting can come from environment variables instead of a +config file (Git 2.31 or newer), next to the variables that point package +managers at the proxy. Each `GIT_CONFIG_KEY_n` and `GIT_CONFIG_VALUE_n` pair is +one `git config` setting, and `GIT_CONFIG_COUNT` says how many pairs there are. +This sends both upstreams from the example above through the proxy: + +```bash +export GIT_CONFIG_COUNT=2 +export GIT_CONFIG_KEY_0=url.http://localhost:8080/git/github/.insteadOf +export GIT_CONFIG_VALUE_0=https://github.com/ +export GIT_CONFIG_KEY_1=url.http://localhost:8080/git/wikimedia/.insteadOf +export GIT_CONFIG_VALUE_1=https://gerrit.wikimedia.org/r/ +``` + +Every process the job starts inherits these variables, so the rewrite also +applies when a package manager runs `git` for a Git dependency, as pip, +Bundler, Composer, SwiftPM, and Go for modules it fetches directly do. Those +fetches are cached too. npm downloads GitHub dependencies as tarballs over +HTTPS where it can, without `git`, so the rewrite does not reach them. + `repos` patterns use `path.Match` syntax, where `*` does not cross a `/`; a pattern ending in `/**` also matches every repository nested below its prefix, so `mediawiki/**` matches `mediawiki/skins/Vector`. An upstream From eb9216528fb9e27becfcc7d51d375607401ab2ac Mon Sep 17 00:00:00 2001 From: montehurd Date: Mon, 19 Oct 2026 13:15:07 -0700 Subject: [PATCH 3/9] Note that upstream.auth credentials reach every route to a host Credentials are matched by URL, so a token configured for a Git upstream is also sent by a generic upstream at the same URL, which forwards any path and checks no repos allowlist. With a token for a private GitHub org, /generic/github/acme/private.git/info/refs would list that repository's refs to anyone who can reach the proxy. --- docs/configuration.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3942e31b..a84752bf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -683,7 +683,12 @@ without `repos` serves every repository on it, and a repository outside `repos` answers 404. When `upstream.auth` has credentials for an upstream, `repos` is required: everyone who can reach the proxy can read every repository it serves, so list only those they may read. Per-client -permissions are not supported. +permissions are not supported. `upstream.auth` matches credentials by URL, so +they are sent by every route to that host, not only `/git`. With a token for +`https://github.com`, a `generic` upstream at the same URL, such as the one +the mise setup in the README uses, sends it for any path and checks no +`repos`, so `/generic/github/acme/private.git/info/refs?service=git-upload-pack` +would list a private repository's refs to anyone. Every request for a repository's refs checks the upstream's branches and tags, and fetches only when they changed, so a clone right after a push sees the new From f3e4e097efccf1443ac3caa8861cb13df954fa23 Mon Sep 17 00:00:00 2001 From: montehurd Date: Mon, 19 Oct 2026 13:16:44 -0700 Subject: [PATCH 4/9] Cap concurrent upload-pack processes and request body size Every POST git-upload-pack started a pack-objects with git's defaults, a delta-search thread per CPU and unbounded window memory, and nothing limited how many ran at once, so a burst of CI clones of one large repository could run the proxy out of memory. At most two upload-pack processes per CPU (GOMAXPROCS, so a container's limit) now build packs at once, and further requests wait for a slot while the client is connected. ls-refs builds no pack and does not wait. Serving runs with pack.threads=1 and pack.windowMemory=64m, and every git process with core.bigFileThreshold=8m, so index-pack streams large files instead of holding them in memory. A request body was streamed to upload-pack unbounded, so a small gzip body could feed it an arbitrarily long negotiation for as long as the request may run. Bodies are now limited to 100MB, both as received and decoded, and one past the limit fails with an error instead of being cut short quietly. --- docs/configuration.md | 8 +- internal/gitcache/cache.go | 23 +++++- internal/gitcache/git.go | 4 + internal/gitcache/serve.go | 84 +++++++++++++++++++- internal/gitcache/serve_test.go | 131 ++++++++++++++++++++++++++++++++ 5 files changed, 244 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a84752bf..0f39c943 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -732,7 +732,13 @@ that does not fit is refused, and the repository kept. Least recently used repositories are evicted after each fetch that takes the cache past `max_size`, never one a request is reading. A transfer that receives nothing for 10 minutes is abandoned. Git's automatic garbage -collection runs as part of fetches. +collection runs as part of fetches, and files over 8MB are kept without delta +compression, so one large file does not take a fetch's memory. + +Packs for clients are built by at most two `upload-pack` processes per CPU +the proxy may use, each with one delta-search thread and 64MB of delta +window, and further clones wait for one to finish; listing refs does not +wait. An `upload-pack` request is limited to 100MB, before and after gzip. Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so repointing an upstream starts a fresh cache, and the old URL's directory is diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index eee4d34b..f575db2f 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -16,6 +16,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "slices" "strings" "sync" @@ -63,9 +64,12 @@ type Options struct { // else in CacheDir is touched, so it may hold other data. Upstreams map[string]string - // refCheckTimeout and lowSpeedTime override their defaults in tests. + // refCheckTimeout, lowSpeedTime, maxRequestBody and maxUploadPacks + // override their defaults in tests. refCheckTimeout time.Duration lowSpeedTime int + maxRequestBody int64 + maxUploadPacks int } // Cache is the Git repository cache. internal/handler serves it at /git. @@ -77,8 +81,10 @@ type Cache struct { dir string relay *relay fetches chan struct{} + uploadPacks chan struct{} refCheckTimeout time.Duration lowSpeedTime int + maxRequestBody int64 ctx context.Context cancel context.CancelFunc @@ -117,6 +123,12 @@ type repo struct { // start any number and hang up. const maxConcurrentFetches = 16 +// uploadPacksPerCPU caps the upload-pack processes building packs at once, +// so a burst of clones cannot run the proxy out of memory. Each uses one +// delta-search thread (servingConfig); twice the CPUs keeps them busy while +// some packs wait on slow clients. +const uploadPacksPerCPU = 2 + // New creates the cache, loading repositories already in CacheDir. func New(opts Options) (*Cache, error) { gitPath, err := exec.LookPath("git") @@ -135,6 +147,10 @@ func New(opts Options) (*Cache, error) { client.Timeout = 0 client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + uploadPacks := uploadPacksPerCPU * runtime.GOMAXPROCS(0) + if opts.maxUploadPacks > 0 { + uploadPacks = opts.maxUploadPacks + } c := &Cache{ opts: opts, client: &client, @@ -142,8 +158,10 @@ func New(opts Options) (*Cache, error) { gitPath: gitPath, dir: dir, fetches: make(chan struct{}, maxConcurrentFetches), + uploadPacks: make(chan struct{}, uploadPacks), refCheckTimeout: defaultRefCheckTimeout, lowSpeedTime: defaultLowSpeedTime, + maxRequestBody: defaultMaxRequestBody, repos: make(map[string]*repo), tooLarge: make(map[string]time.Time), failedWants: make(map[string]time.Time), @@ -154,6 +172,9 @@ func New(opts Options) (*Cache, error) { if opts.lowSpeedTime > 0 { c.lowSpeedTime = opts.lowSpeedTime } + if opts.maxRequestBody > 0 { + c.maxRequestBody = opts.maxRequestBody + } if err := c.load(); err != nil { return nil, err } diff --git a/internal/gitcache/git.go b/internal/gitcache/git.go index 736ee8a6..0cc2c11a 100644 --- a/internal/gitcache/git.go +++ b/internal/gitcache/git.go @@ -43,6 +43,10 @@ var gitConfig = [][2]string{ // receives nothing for that long, so a connection that stalls mid-pack // cannot hold a refresh for fetchTimeout. {"http.lowSpeedLimit", "1"}, + // index-pack streams blobs larger than this instead of holding them in + // memory, and pack-objects stores them without delta compression, so a + // large file costs a fetch or a clone little memory. + {"core.bigFileThreshold", "8m"}, } // command builds a git process with an environment of its own: none of the diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index b6fab6ea..6d88df46 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -1,6 +1,7 @@ package gitcache import ( + "bytes" "compress/gzip" "context" "errors" @@ -8,7 +9,9 @@ import ( "net/http" "path/filepath" "regexp" + "slices" "strings" + "sync/atomic" ) // gitProtocolPattern is what a Git-Protocol header may hold, such as @@ -86,16 +89,26 @@ func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { // upstream first. func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { c := l.cache - body := io.Reader(r.Body) + // The raw body is limited too: a gzip stream of any length can decode + // to almost nothing. + raw := &bodyLimit{r: r.Body, remaining: c.maxRequestBody} + decoded := raw if r.Header.Get("Content-Encoding") == "gzip" { - gz, err := gzip.NewReader(r.Body) + gz, err := gzip.NewReader(raw) if err != nil { http.Error(w, "invalid gzip body", http.StatusBadRequest) return } defer func() { _ = gz.Close() }() - body = gz + decoded = &bodyLimit{r: gz, remaining: c.maxRequestBody} } + defer func() { + if raw.exceeded.Load() || decoded.exceeded.Load() { + c.logger.Warn("git cache: request body passed the limit", + "repository", l.repo.key, "limit", c.maxRequestBody) + } + }() + body := io.Reader(decoded) if l.fetchWants { var wants []string wants, body = readWants(body) @@ -104,11 +117,40 @@ func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { c.logger.Warn("git cache: fetching wanted objects failed", "repository", l.repo.key, "error", err) } } + var command string + command, body = requestCommand(body) + // ls-refs builds no pack, so it does not wait behind clones. + if command != "ls-refs" { + select { + case c.uploadPacks <- struct{}{}: + defer func() { <-c.uploadPacks }() + case <-r.Context().Done(): + return + } + } setNoCache(w) w.Header().Set("Content-Type", resultType) c.uploadPack(w, r, l.repo, gitProtocol(r), body) } +// requestCommand returns the protocol v2 command an upload-pack request +// starts with, such as "fetch", or "" for an earlier protocol, with a reader +// that replays the whole body. +func requestCommand(body io.Reader) (string, io.Reader) { + var consumed bytes.Buffer + _, line, _ := newPktReader(io.TeeReader(body, &consumed)).next() + command, ok := strings.CutPrefix(line, "command=") + if !ok { + command = "" + } + return command, io.MultiReader(&consumed, body) +} + +// servingConfig gives each pack-objects that upload-pack runs one +// delta-search thread with a bounded window, where git defaults to a thread +// per CPU and no bound. +var servingConfig = []string{"-c", "pack.threads=1", "-c", "pack.windowMemory=64m"} + // uploadPack runs git upload-pack on the cached repository in stateless mode, // the way git http-backend does. func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, protocol string, stdin io.Reader, args ...string) { @@ -116,7 +158,7 @@ func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, pro if protocol != "" { env = append(env, "GIT_PROTOCOL="+protocol) } - args = append(append([]string{"upload-pack", "--stateless-rpc"}, args...), rp.dir) + args = slices.Concat(servingConfig, []string{"upload-pack", "--stateless-rpc"}, args, []string{rp.dir}) cmd := c.command(r.Context(), env, args...) cmd.Stdin = stdin cmd.Stdout = w @@ -128,6 +170,40 @@ func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, pro } } +// defaultMaxRequestBody bounds an upload-pack request far above any real +// negotiation, so a small gzip body cannot feed upload-pack for as long as +// the request may run. +const defaultMaxRequestBody = 100 << 20 + +var errRequestTooLarge = errors.New("request body too large") + +// bodyLimit fails reads past remaining bytes, where io.LimitReader would end +// the body quietly and leave upload-pack a request cut short. +type bodyLimit struct { + r io.Reader + remaining int64 + exceeded atomic.Bool +} + +func (b *bodyLimit) Read(p []byte) (int, error) { + if b.exceeded.Load() { + return 0, errRequestTooLarge + } + // One byte past the limit tells a longer body from one that ends at it. + if int64(len(p)) > b.remaining+1 { + p = p[:b.remaining+1] + } + n, err := b.r.Read(p) + if int64(n) <= b.remaining { + b.remaining -= int64(n) + return n, err + } + n = int(b.remaining) + b.remaining = 0 + b.exceeded.Store(true) + return n, errRequestTooLarge +} + func gitProtocol(r *http.Request) string { if protocol := r.Header.Get("Git-Protocol"); gitProtocolPattern.MatchString(protocol) { return protocol diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go index b81370e4..b5779ae2 100644 --- a/internal/gitcache/serve_test.go +++ b/internal/gitcache/serve_test.go @@ -7,10 +7,12 @@ import ( "io" "net/http" "os/exec" + "path/filepath" "slices" "strings" "sync" "testing" + "time" ) // TestRequestBodies sends the bodies git sends for large requests: gzipped, @@ -53,6 +55,135 @@ func TestRequestBodies(t *testing.T) { } } +// TestRequestBodyLimit refuses a request past the limit, before or after +// gzip, including one that is small compressed. +func TestRequestBodyLimit(t *testing.T) { + const limit = 4096 + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxRequestBody = limit }) + tc.clone(t, "repo") + + lsRefs := func(prefixes int) string { + return pktLine("command=ls-refs\n") + pktDelim + + strings.Repeat(pktLine("ref-prefix refs/heads/\n"), prefixes) + pktFlush + } + gzipped := func(body string) *bytes.Buffer { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = io.WriteString(gz, body) + _ = gz.Close() + return &buf + } + small, large := lsRefs(10), lsRefs(200) + if len(small) > limit || len(large) <= limit { + t.Fatalf("bodies of %d and %d bytes do not straddle the limit", len(small), len(large)) + } + compressed := gzipped(large) + if compressed.Len() > limit { + t.Fatalf("compressed body is %d bytes, over the limit", compressed.Len()) + } + // Empty gzip members before a small body: long on the wire, short decoded. + var padded bytes.Buffer + for empty := gzipped(""); padded.Len() <= limit; { + padded.Write(empty.Bytes()) + } + padded.Write(gzipped(small).Bytes()) + for _, tt := range []struct { + name string + body io.Reader + encoding string + served bool + }{ + {"small", strings.NewReader(small), "", true}, + {"small gzip", gzipped(small), "gzip", true}, + {"large", strings.NewReader(large), "", false}, + {"large gzip", compressed, "gzip", false}, + {"padded gzip", &padded, "gzip", false}, + } { + req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + if tt.encoding != "" { + req.Header.Set("Content-Encoding", tt.encoding) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if served := strings.Contains(string(out), "refs/heads/main"); served != tt.served { + t.Errorf("%s: served %t, want %t (status %d, body %q)", tt.name, served, tt.served, resp.StatusCode, out) + } + } +} + +// TestUploadPackSlots holds the only upload-pack slot: a clone waits for it, +// while listing refs, which builds no pack, does not. +func TestUploadPackSlots(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.maxUploadPacks = 1 }) + tc.clone(t, "repo") + + start := func(args ...string) <-chan error { + done := make(chan error, 1) + go func() { + out, err := tryGit(t, "", args...) + if err != nil { + err = fmt.Errorf("%w\n%s", err, out) + } + done <- err + }() + return done + } + + tc.uploadPacks <- struct{}{} + held := true + release := func() { + if held { + held = false + <-tc.uploadPacks + } + } + // A failure while the slot is held must free it for the cache to close. + defer release() + select { + case err := <-start("-c", "protocol.version=2", "ls-remote", tc.url("repo")): + if err != nil { + t.Fatalf("ls-remote: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("ls-remote waited for the slot") + } + clone := start("clone", "--quiet", tc.url("repo"), filepath.Join(t.TempDir(), "clone")) + select { + case err := <-clone: + t.Fatalf("clone finished while the slot was held: %v", err) + case <-time.After(time.Second): + } + release() + if err := <-clone; err != nil { + t.Errorf("clone: %v", err) + } +} + +func TestRequestCommand(t *testing.T) { + for _, tt := range []struct{ body, want string }{ + {pktLine("command=ls-refs\n") + pktDelim + pktFlush, "ls-refs"}, + {pktLine("command=fetch\n") + pktDelim + pktLine("done\n") + pktFlush, "fetch"}, + {pktLine("want "+strings.Repeat("a", 40)+"\n") + pktFlush, ""}, + {"", ""}, + } { + command, body := requestCommand(strings.NewReader(tt.body)) + replayed, _ := io.ReadAll(body) + if command != tt.want || string(replayed) != tt.body { + t.Errorf("requestCommand(%q) = %q, %q; want %q and the whole body", tt.body, command, replayed, tt.want) + } + } +} + // TestCloneWithChunkedRequests clones a repository with enough refs that // git's requests, to the cache and from the cache to the upstream, outgrow // http.postBuffer and are sent chunked. From 7c204b5f137fc91237db3d56e764cb33cb0472e8 Mon Sep 17 00:00:00 2001 From: montehurd Date: Mon, 19 Oct 2026 13:16:55 -0700 Subject: [PATCH 5/9] Apply refresh_wait to the ref check only refresh_wait cut off the whole refresh, fetch included, so after a push with a large pack a client got the old refs once it passed, though the upstream was fine and the docs say a clone right after a push sees the new commit. It now bounds only the check of the upstream's branches and tags. Once the check finds they changed, requests wait for the fetch; one that fails, or stalls and is abandoned by http.lowSpeedTime, still falls back to the cached copy within stale_max_age. --- config.example.yaml | 2 +- docs/configuration.md | 13 +++--- internal/config/git.go | 6 +-- internal/gitcache/cache_test.go | 37 ++++++++++++++++ internal/gitcache/helpers_test.go | 7 ++- internal/gitcache/refresh.go | 71 ++++++++++++++++++++++--------- 6 files changed, 106 insertions(+), 30 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 1a0ea2f2..f1770862 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -380,7 +380,7 @@ upstream: # cache_dir: ./cache/git # local disk, used by this process only # max_size: "" # evict least recently used repositories past this # max_repo_size: 5GB # refuse a repository that grows past this -# refresh_wait: 5s # then serve the cached copy if the upstream is slow +# refresh_wait: 5s # then serve the cached copy if the upstream check is slow # stale_max_age: 24h # never serve a copy not checked for this long # Gradle HttpBuildCache configuration diff --git a/docs/configuration.md b/docs/configuration.md index 0f39c943..8900262d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -641,7 +641,7 @@ git: cache_dir: /data/git # local disk, used by this process only max_size: 50GB # evict least recently used repositories past this max_repo_size: 5GB # refuse a repository that grows past this - refresh_wait: 5s # then serve the cached copy if the upstream is slow + refresh_wait: 5s # then serve the cached copy if the upstream check is slow stale_max_age: 24h # never serve a copy not checked for this long ``` @@ -708,10 +708,13 @@ through any other, so a fetch by ID could reach repositories outside `repos`. Any commit the cache holds can be fetched by ID, including one a force-push left behind, as from the upstream. -If the upstream fails or takes longer than `refresh_wait`, a repository checked -successfully within `stale_max_age` is served from the cache while the refresh -continues. A `stale_max_age` of `0` never serves a copy the upstream has not -just confirmed. A repository the upstream reports missing (404), refuses (401, 403) +If the upstream fails, or its check of branches and tags takes longer than +`refresh_wait`, a repository checked successfully within `stale_max_age` is +served from the cache while the refresh continues. Once the check finds new +commits, the request waits for their fetch, so a clone right after a push gets +them; if the fetch fails or stalls, the cached copy is served. A +`stale_max_age` of `0` never serves a copy the upstream has not just +confirmed. A repository the upstream reports missing (404), refuses (401, 403) or redirects is not served from the cache. Redirects are never followed, so point the upstream at a repository's current location. diff --git a/internal/config/git.go b/internal/config/git.go index 8a0028d2..1924c1d5 100644 --- a/internal/config/git.go +++ b/internal/config/git.go @@ -92,9 +92,9 @@ type GitConfig struct { // would pass it fails. "0" disables the limit. Default: 5GB. MaxRepoSize string `json:"max_repo_size" yaml:"max_repo_size"` - // RefreshWait is how long a request waits for a repository to be - // refreshed from upstream before it is served the cached copy. - // Default: 5s. + // RefreshWait is how long a request waits for the upstream's branches + // and tags to be checked before it is served the cached copy. A fetch of + // new commits the check finds is waited for in full. Default: 5s. RefreshWait string `json:"refresh_wait" yaml:"refresh_wait"` // StaleMaxAge is how long after the last successful upstream check a diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index f5d14b2b..35b0a201 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -232,6 +232,43 @@ func TestSlowUpstreamServesCachedCopyAfterRefreshWait(t *testing.T) { waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) } +// TestSlowFetchAfterPushIsWaitedFor has the upstream answer the ref check at +// once and the fetch after a push take longer than RefreshWait: a clone +// still gets the pushed commit. RefreshWait leaves the check time to finish +// on a slow machine. +func TestSlowFetchAfterPushIsWaitedFor(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 2 * time.Second }) + tc.clone(t, "repo") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + upstream.delayFetches.Store(int64(4 * time.Second)) + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want the pushed %s", got, head) + } +} + +// TestStalledFetchAfterPushServesCachedCopy has the fetch after a push +// stall: once it is abandoned, the clone gets the cached copy. +func TestStalledFetchAfterPushServesCachedCopy(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { + o.RefreshWait = 100 * time.Millisecond + o.lowSpeedTime = 1 + }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + upstream.stallFetches.Store(true) + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { + t.Errorf("HEAD = %s, want the cached %s", got, old) + } +} + func TestUpstreamCredentialsReachEveryUpstreamRequest(t *testing.T) { upstream := newFakeUpstream(t) upstream.requireAuth = "Bearer secret" diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index ce6becae..9f558813 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -91,8 +91,10 @@ type fakeUpstream struct { gitSessions atomic.Int32 unauthed atomic.Int32 - // stallFetches makes git's own upload-pack requests hang. + // stallFetches makes git's own upload-pack requests hang, and + // delayFetches holds them for that long. stallFetches atomic.Bool + delayFetches atomic.Int64 bodiesMu sync.Mutex bodies []string @@ -170,6 +172,9 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { if u.v0Only { r.Header.Del("Git-Protocol") } + if delay := u.delayFetches.Load(); delay > 0 && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { + time.Sleep(time.Duration(delay)) + } if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { // The server notices the client leaving only once the body is read. _, _ = io.Copy(io.Discard, r.Body) diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 801e8b3f..dd9af959 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/git-pkgs/proxy/internal/metrics" @@ -38,13 +39,32 @@ var errClosed = errors.New("git cache is closed") type refreshCall struct { done chan struct{} err error + // checked is closed once the upstream's branches and tags have been + // compared with the cached ones, and fetching then says whether they + // differ, so a fetch follows. + checked chan struct{} + fetching bool + checkedOnce sync.Once +} + +func newRefreshCall() *refreshCall { + return &refreshCall{done: make(chan struct{}), checked: make(chan struct{})} +} + +func (call *refreshCall) markChecked(fetching bool) { + call.checkedOnce.Do(func() { + call.fetching = fetching + close(call.checked) + }) } // ensureFresh brings the repository up to date with its upstream before it is // served. A repository checked successfully within StaleMaxAge is served -// from the cache if the upstream fails or takes longer than RefreshWait; the -// refresh carries on in the background either way. One never cloned, or not -// checked within StaleMaxAge, is served only once the refresh succeeds. +// from the cache if the upstream fails or its check takes longer than +// RefreshWait; the refresh carries on in the background either way. Once the +// check finds new commits, their fetch is waited for, so a clone right after +// a push gets them. One never cloned, or not checked within StaleMaxAge, is +// served only once the refresh succeeds. func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error { call := c.startRefresh(rp, repoURL) checked := time.Unix(0, rp.lastChecked.Load()) @@ -59,20 +79,28 @@ func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error timer := time.NewTimer(c.opts.RefreshWait) defer timer.Stop() - select { - case <-call.done: - if call.err != nil && servesStale(call.err) { - c.logger.Warn("git cache: refresh failed, serving cached copy", - "repository", rp.key, "checked", checked, "error", call.err) + refChecked, timeout := call.checked, timer.C + for { + select { + case <-refChecked: + refChecked = nil + if call.fetching { + timeout = nil + } + case <-call.done: + if call.err != nil && servesStale(call.err) { + c.logger.Warn("git cache: refresh failed, serving cached copy", + "repository", rp.key, "checked", checked, "error", call.err) + return nil + } + return call.err + case <-timeout: + c.logger.Info("git cache: upstream check still running, serving cached copy", + "repository", rp.key, "checked", checked) return nil + case <-ctx.Done(): + return ctx.Err() } - return call.err - case <-timer.C: - c.logger.Info("git cache: refresh still running, serving cached copy", - "repository", rp.key, "checked", checked) - return nil - case <-ctx.Done(): - return ctx.Err() } } @@ -90,7 +118,7 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { if rp.refreshing != nil { return rp.refreshing } - call := &refreshCall{done: make(chan struct{})} + call := newRefreshCall() if c.closed { call.err = errClosed close(call.done) @@ -103,7 +131,8 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { defer c.refreshes.Done() ctx, cancel := context.WithTimeout(c.ctx, fetchTimeout) start := time.Now() - call.err = c.refresh(ctx, rp, repoURL) + call.err = c.refresh(ctx, rp, repoURL, call.markChecked) + call.markChecked(false) cancel() metrics.RecordUpstreamFetch(ecosystem, time.Since(start)) // A refresh cut short by Close is not an upstream failure. @@ -125,7 +154,8 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { // refresh checks the upstream's branches and tags and fetches if they differ // from the cached ones, cloning the repository if it is not cached yet. -func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string) error { +// checked is called once a cached repository's refs have been compared. +func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked func(fetching bool)) error { if c.isTooLarge(rp.key) { return ErrTooLarge } @@ -136,7 +166,7 @@ func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string) error { return err } if rp.exists() { - err = c.update(ctx, rp, repoURL, remote) + err = c.update(ctx, rp, repoURL, remote, checked) } else { err = c.clone(ctx, rp, repoURL, remote) } @@ -187,7 +217,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo // update fetches when the upstream's branches or tags differ from the cached // ones, and follows a change of the upstream's default branch. -func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { +func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote remoteRefs, checked func(fetching bool)) error { unlock, err := rp.lockWrites(ctx) if err != nil { return err @@ -198,6 +228,7 @@ func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote rem return err } fetched := !maps.Equal(local, remote.refs) + checked(fetched) if fetched { limit, err := c.fetchLimit(rp) if err != nil { From 64a9c6307ec900d7fe3cec5416f505796b16d57c Mon Sep 17 00:00:00 2001 From: montehurd Date: Mon, 19 Oct 2026 13:17:04 -0700 Subject: [PATCH 6/9] Lock cache_dir against a second process The cache clears its temporary directory at startup, so a second process given the same cache_dir, such as a second pod on a shared volume, would delete the first one's clones in progress, and each process's eviction knows only its own readers. New now locks {cache_dir}/.gitcache-lock before touching anything and holds it until Close: flock on Unix, a handle opened without sharing on Windows. The OS releases it when the process exits, so a crashed process can restart. A second process fails to start with an error naming the directory. --- deploy/charts/proxy/values.yaml | 1 + docs/configuration.md | 4 +++- internal/config/git.go | 2 +- internal/gitcache/cache.go | 23 ++++++++++++++++++++++- internal/gitcache/cache_test.go | 29 +++++++++++++++++++++++++++++ internal/gitcache/lock_unix.go | 24 ++++++++++++++++++++++++ internal/gitcache/lock_windows.go | 27 +++++++++++++++++++++++++++ 7 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 internal/gitcache/lock_unix.go create mode 100644 internal/gitcache/lock_windows.go diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index 38617849..9a74e475 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -1,6 +1,7 @@ # More than one replica requires config.data.database on Postgres, # config.data.storage on object storage, deploymentStrategy: RollingUpdate, # and a PVC access mode other than ReadWriteOnce (or persistence disabled). +# With upstream.git, each replica also needs its own git.cache_dir. replicaCount: 1 nameOverride: "" diff --git a/docs/configuration.md b/docs/configuration.md index 8900262d..8a3cdac6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -747,7 +747,9 @@ Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so repointing an upstream starts a fresh cache, and the old URL's directory is removed at startup. Nothing else in `cache_dir` is touched, and the directories of upstreams no longer configured are only logged, so remove -those yourself. +those yourself. The proxy locks `cache_dir` while it runs, through a +`.gitcache-lock` file in it, so a second process given the same directory, +such as another replica on a shared volume, fails to start; give each its own. Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`, diff --git a/internal/config/git.go b/internal/config/git.go index 1924c1d5..8fe72ea2 100644 --- a/internal/config/git.go +++ b/internal/config/git.go @@ -81,7 +81,7 @@ func matchRepoPattern(pattern, repo string) bool { // GitConfig configures the Git repository cache behind /git/{name}/. type GitConfig struct { // CacheDir holds the cached bare repositories. It must be local disk - // used by this process only. Default: ./cache/git. + // used by this process only, which locks it. Default: ./cache/git. CacheDir string `json:"cache_dir" yaml:"cache_dir"` // MaxSize evicts the least recently used repositories once their total diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index f575db2f..2924c089 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -32,6 +32,10 @@ const ( // with an upstream's directory, and the name is specific in case // cache_dir is shared with other data. tmpDirName = ".gitcache-tmp" + // lockFileName is locked while the cache is open, so a second process + // given the same cache_dir fails to start instead of deleting the first + // one's clones in progress. + lockFileName = ".gitcache-lock" // usedStampInterval limits how often a repository's last use is written // to disk. usedStampInterval = time.Minute @@ -44,6 +48,7 @@ const ( checkedStamp = "proxy-checked" usedStamp = "proxy-used" dirPermissions = 0o750 + lockFilePermissions = 0o600 ) // Options configures a Cache. @@ -79,6 +84,7 @@ type Cache struct { logger *slog.Logger gitPath string dir string + lock *os.File relay *relay fetches chan struct{} uploadPacks chan struct{} @@ -118,6 +124,8 @@ type repo struct { usedWritten atomic.Int64 } +var errLocked = errors.New("locked by another process") + // maxConcurrentFetches caps clones and fetches running at once. Refreshes // outlive the requests that start them, so without a cap a client could // start any number and hang up. @@ -139,6 +147,16 @@ func New(opts Options) (*Cache, error) { if err != nil { return nil, err } + if err := os.MkdirAll(dir, dirPermissions); err != nil { + return nil, err + } + lock, err := lockFile(filepath.Join(dir, lockFileName)) + if errors.Is(err, errLocked) { + return nil, fmt.Errorf("git.cache_dir %s is in use by another process; each needs its own", dir) + } + if err != nil { + return nil, fmt.Errorf("locking git.cache_dir %s: %w", dir, err) + } logger := opts.Logger if logger == nil { logger = slog.New(slog.DiscardHandler) @@ -157,6 +175,7 @@ func New(opts Options) (*Cache, error) { logger: logger, gitPath: gitPath, dir: dir, + lock: lock, fetches: make(chan struct{}, maxConcurrentFetches), uploadPacks: make(chan struct{}, uploadPacks), refCheckTimeout: defaultRefCheckTimeout, @@ -176,9 +195,11 @@ func New(opts Options) (*Cache, error) { c.maxRequestBody = opts.maxRequestBody } if err := c.load(); err != nil { + _ = lock.Close() return nil, err } if c.relay, err = newRelay(c.client, logger); err != nil { + _ = lock.Close() return nil, fmt.Errorf("starting git relay: %w", err) } c.ctx, c.cancel = context.WithCancel(context.Background()) @@ -201,7 +222,7 @@ func (c *Cache) Close() error { c.mu.Unlock() c.cancel() c.refreshes.Wait() - return c.relay.close() + return errors.Join(c.relay.close(), c.lock.Close()) } // UpstreamDir names the top-level cache directory of the upstream called diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 35b0a201..7be2f927 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -437,6 +437,9 @@ func TestNewLoadsCachedRepositories(t *testing.T) { if err := os.MkdirAll(filepath.Join(tc.tmpDir(), "clone-leftover"), 0o750); err != nil { t.Fatal(err) } + if err := tc.Close(); err != nil { + t.Fatal(err) + } reloaded, err := New(Options{ Upstreams: map[string]string{"up": "up"}, @@ -683,6 +686,32 @@ func TestCloseWaitsForRefreshes(t *testing.T) { } } +// TestCacheDirInUse opens a second cache on a directory one already uses, +// as a second pod on a shared volume would, then again once the first has +// closed. +func TestCacheDirInUse(t *testing.T) { + dir := t.TempDir() + open := func() (*Cache, error) { return New(Options{CacheDir: dir, Client: &http.Client{}}) } + first, err := open() + if err != nil { + t.Fatal(err) + } + if second, err := open(); err == nil { + _ = second.Close() + t.Fatal("a second cache opened on the same directory") + } else if !strings.Contains(err.Error(), "in use by another process") { + t.Errorf("error = %v, want one naming the directory as in use", err) + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + again, err := open() + if err != nil { + t.Fatalf("reopening after Close: %v", err) + } + _ = again.Close() +} + // TestCloseTwice checks closing again does nothing, as a cleanup that also // closes the cache may. func TestCloseTwice(t *testing.T) { diff --git a/internal/gitcache/lock_unix.go b/internal/gitcache/lock_unix.go new file mode 100644 index 00000000..215027a1 --- /dev/null +++ b/internal/gitcache/lock_unix.go @@ -0,0 +1,24 @@ +//go:build !windows + +package gitcache + +import ( + "errors" + "os" + "syscall" +) + +func lockFile(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, lockFilePermissions) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, errLocked + } + return nil, err + } + return f, nil +} diff --git a/internal/gitcache/lock_windows.go b/internal/gitcache/lock_windows.go new file mode 100644 index 00000000..99f46d51 --- /dev/null +++ b/internal/gitcache/lock_windows.go @@ -0,0 +1,27 @@ +package gitcache + +import ( + "errors" + "os" + "syscall" +) + +const errorSharingViolation syscall.Errno = 32 + +// lockFile opens path with no sharing, so no other handle can open it while +// this one is open. +func lockFile(path string) (*os.File, error) { + name, err := syscall.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + h, err := syscall.CreateFile(name, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, + syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if errors.Is(err, errorSharingViolation) { + return nil, errLocked + } + if err != nil { + return nil, err + } + return os.NewFile(uintptr(h), path), nil +} From 09ce83d8c5e95db9b96a5174593b7011984dcb43 Mon Sep 17 00:00:00 2001 From: montehurd Date: Tue, 20 Oct 2026 09:40:31 -0700 Subject: [PATCH 7/9] Evict again once repositories in use are released Eviction skips repositories a request or refresh is using, and only the release of one too large to refresh ran it again, so a cache filled past max_size while its repositories were in use stayed over it once they went idle, until the next fetch. Eviction now records when it is left over max_size, and the next release runs it again. --- internal/gitcache/cache.go | 14 ++++++++++---- internal/gitcache/cache_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 2924c089..a1cbcdb8 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -102,6 +102,9 @@ type Cache struct { closed bool repos map[string]*repo tooLarge map[string]time.Time + // overBudget records that eviction left the cache over MaxSize because + // the rest of it was in use. + overBudget bool // failedWants records, by repository key and object ID, wants the // upstream could not provide. failedWants map[string]time.Time @@ -348,8 +351,9 @@ func (c *Cache) release(rp *repo) { delete(c.repos, rp.key) } _, tooLarge := c.tooLarge[rp.key] + retry := tooLarge || c.overBudget c.mu.Unlock() - if idle && exists && tooLarge { + if idle && exists && retry { c.evict() } } @@ -377,9 +381,10 @@ func (rp *repo) exists() bool { // the cache fits in MaxSize, and any that could not be refreshed within // MaxRepoSize, so the next request clones them again with only their // branches and tags. It runs after every fetch, since only fetches grow the -// cache, and when a repository too large to refresh is released. A -// repository is moved aside under the lock, so a request that comes after it -// clones it again instead of reading it while it is deleted. +// cache, and when a repository it had to skip is released: one too large to +// refresh, or any while the cache is over MaxSize. A repository is moved +// aside under the lock, so a request that comes after it clones it again +// instead of reading it while it is deleted. func (c *Cache) evict() { c.mu.Lock() var total int64 @@ -420,6 +425,7 @@ func (c *Cache) evict() { metrics.RecordArtifactEvicted(reason, ecosystem) c.logger.Info("git cache: evicted repository", "repository", rp.key, "size", rp.size.Load(), "reason", reason) } + c.overBudget = c.opts.MaxSize > 0 && total > c.opts.MaxSize c.mu.Unlock() for _, trash := range removed { diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 7be2f927..02e3e582 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -594,6 +594,37 @@ func TestFetchEvictsToMaxSize(t *testing.T) { } } +// TestReleaseEvictsWhatEvictionHadToSkip fills the cache past MaxSize while +// every repository is in use, so eviction after the fetch can remove none. +// The first release brings the cache back within MaxSize. +func TestReleaseEvictsWhatEvictionHadToSkip(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("first") + upstream.newRepo("second") + size := repoSize(t, upstream, "first") + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) + first := tc.acquire("up/first.git") + tc.clone(t, "first") + second := tc.acquire("up/second.git") + defer tc.release(second) + tc.clone(t, "second") + waitFor(t, func() bool { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.overBudget + }) + + tc.release(first) + if fileExists(first.dir) { + t.Error("first is still cached once released") + } + if !fileExists(second.dir) { + t.Error("evicted second while in use") + } + tc.clone(t, "second") +} + // TestNewRemovesRepointedUpstreams starts a cache whose directory holds an // upstream's cache at its old URL, and one of an upstream no longer // configured. Only the first is known to be the cache's to remove. From 160305b83d99ce4e76c69597ca670ea2a6954651 Mon Sep 17 00:00:00 2001 From: montehurd Date: Tue, 20 Oct 2026 11:34:30 -0700 Subject: [PATCH 8/9] Cap git.max_repo_size at git.max_size A repository larger than max_size could still be cloned, up to max_repo_size, which defaults to 5GB whatever max_size is. Once released it was evicted to bring the cache back within max_size, and the next request cloned it again. max_size now caps max_repo_size, so such a repository is refused for an hour like any other too large one, with an error that names both limits. --- docs/configuration.md | 4 ++- internal/config/git.go | 3 ++- internal/gitcache/cache.go | 5 ++++ internal/gitcache/cache_test.go | 47 ++++++++++++++++++++------------- internal/gitcache/upstream.go | 5 ++-- internal/handler/git.go | 2 +- 6 files changed, 42 insertions(+), 24 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8a3cdac6..2af99e1b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -725,7 +725,9 @@ upstream client as every other request, so the address checks (`allow_loopback`, `allow_private_hosts`), `upstream.auth` and the access log apply, and credentials are never passed to `git`. -`max_repo_size` defaults to 5GB, and `max_size` to no limit. A clone or fetch +`max_repo_size` defaults to 5GB, and `max_size` to no limit. `max_size` also +caps `max_repo_size`, since a repository larger than the whole cache would be +evicted as soon as no request was reading it. A clone or fetch that would pass `max_repo_size` is stopped, and nothing of it is kept. A new repository is then refused for an hour. When a refresh of one already cached does not fit, it is served as it is and dropped once no request is reading diff --git a/internal/config/git.go b/internal/config/git.go index 8fe72ea2..d3a22d98 100644 --- a/internal/config/git.go +++ b/internal/config/git.go @@ -89,7 +89,8 @@ type GitConfig struct { MaxSize string `json:"max_size" yaml:"max_size"` // MaxRepoSize is the most one repository may take; a clone or fetch that - // would pass it fails. "0" disables the limit. Default: 5GB. + // would pass it fails. "0" disables the limit. Default: 5GB. MaxSize + // caps it when set. MaxRepoSize string `json:"max_repo_size" yaml:"max_repo_size"` // RefreshWait is how long a request waits for the upstream's branches diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index a1cbcdb8..d65e3df6 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -164,6 +164,11 @@ func New(opts Options) (*Cache, error) { if logger == nil { logger = slog.New(slog.DiscardHandler) } + // A repository larger than the whole cache would be evicted once + // released and cloned again by the next request. + if opts.MaxSize > 0 && (opts.MaxRepoSize <= 0 || opts.MaxRepoSize > opts.MaxSize) { + opts.MaxRepoSize = opts.MaxSize + } client := *opts.Client client.Timeout = 0 client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 02e3e582..1770361b 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -367,27 +367,36 @@ func TestRepositoryOverMaxRepoSizeIsRefused(t *testing.T) { runGit(t, work, "add", "big.bin") runGit(t, work, "commit", "--quiet", "-m", "big") runGit(t, work, "push", "--quiet", "origin", "main") - tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = 64 << 10 }) - resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") - if err != nil { - t.Fatal(err) - } - _ = resp.Body.Close() - if resp.StatusCode != http.StatusInsufficientStorage { - t.Errorf("status %d, want 507", resp.StatusCode) - } - waitForRepos(t, tc.Cache, 0) - if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { - t.Errorf("left %d entries in the temporary directory", len(entries)) - } + // max_size caps max_repo_size. + for name, configure := range map[string]func(*Options){ + "max_repo_size": func(o *Options) { o.MaxRepoSize = 64 << 10 }, + "max_size": func(o *Options) { o.MaxSize = 64 << 10 }, + } { + t.Run(name, func(t *testing.T) { + tc := newTestCache(t, upstream.server.URL, configure) - requests := upstream.requests.Load() - if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { - t.Errorf("second attempt succeeded:\n%s", out) - } - if got := upstream.requests.Load(); got != requests { - t.Errorf("second attempt made %d upstream requests, want none", got-requests) + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusInsufficientStorage { + t.Errorf("status %d, want 507", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { + t.Errorf("left %d entries in the temporary directory", len(entries)) + } + + requests := upstream.requests.Load() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("second attempt succeeded:\n%s", out) + } + if got := upstream.requests.Load(); got != requests { + t.Errorf("second attempt made %d upstream requests, want none", got-requests) + } + }) } } diff --git a/internal/gitcache/upstream.go b/internal/gitcache/upstream.go index a21f39df..83c14349 100644 --- a/internal/gitcache/upstream.go +++ b/internal/gitcache/upstream.go @@ -30,8 +30,9 @@ var ( // copy is not served after one, since the upstream may have withdrawn // access to it. ErrRefused = errors.New("upstream refused the request") - // ErrTooLarge is returned for a repository larger than git.max_repo_size. - ErrTooLarge = errors.New("repository is larger than git.max_repo_size") + // ErrTooLarge is returned for a repository larger than git.max_repo_size, + // or than git.max_size where that is smaller. + ErrTooLarge = errors.New("repository is larger than git.max_repo_size or git.max_size") ) // remoteRefs is what a repository advertises: its branch and tag tips, and diff --git a/internal/handler/git.go b/internal/handler/git.go index 4af07ea0..d7d8dd04 100644 --- a/internal/handler/git.go +++ b/internal/handler/git.go @@ -206,7 +206,7 @@ func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err erro case errors.Is(err, gitcache.ErrRefused): http.Error(w, "upstream refused the request", http.StatusBadGateway) case errors.Is(err, gitcache.ErrTooLarge): - http.Error(w, "repository is larger than git.max_repo_size", http.StatusInsufficientStorage) + http.Error(w, "repository is larger than git.max_repo_size or git.max_size", http.StatusInsufficientStorage) default: http.Error(w, "upstream unavailable", http.StatusBadGateway) } From 82915f6924ab3e0fa7cab69b4bd12944f4ed53e5 Mon Sep 17 00:00:00 2001 From: montehurd Date: Tue, 20 Oct 2026 11:34:30 -0700 Subject: [PATCH 9/9] Count fetches in progress toward git.max_size Fetches enforced only max_repo_size, and eviction ran after each one, so up to 16 concurrent fetches could take the cache past max_size by as much as each may download. Each fetch now reserves space within max_size as the relay receives its bytes, a chunk at a time, evicting least recently used idle repositories to make room. When the fetch ends, the repository's measured size replaces the reservation under the cache lock, so its bytes are neither counted twice nor missed. When evicting every idle repository would not make room, none is evicted for the fetch, and it is cut off with ErrCacheFull rather than ErrTooLarge, so the repository is not refused for an hour: a clone answers 507, and a refresh serves the cached copy as when the upstream fails. A fetch that runs out of room while another has just been cut off waits, up to a minute, for that one's space, so two fetches that do not fit together do not both fail. A fetch that was cut off never waits, so no two wait on each other. --- docs/configuration.md | 33 +++-- internal/config/git.go | 5 +- internal/gitcache/budget.go | 126 +++++++++++++++++ internal/gitcache/budget_test.go | 218 ++++++++++++++++++++++++++++++ internal/gitcache/cache.go | 41 ++++-- internal/gitcache/helpers_test.go | 2 +- internal/gitcache/refresh.go | 23 +++- internal/gitcache/relay.go | 28 +++- internal/gitcache/relay_test.go | 4 +- internal/gitcache/upstream.go | 3 + internal/gitcache/wants.go | 11 +- internal/handler/git.go | 2 + 12 files changed, 452 insertions(+), 44 deletions(-) create mode 100644 internal/gitcache/budget.go create mode 100644 internal/gitcache/budget_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 2af99e1b..b8642a54 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -727,18 +727,27 @@ apply, and credentials are never passed to `git`. `max_repo_size` defaults to 5GB, and `max_size` to no limit. `max_size` also caps `max_repo_size`, since a repository larger than the whole cache would be -evicted as soon as no request was reading it. A clone or fetch -that would pass `max_repo_size` is stopped, and nothing of it is kept. A new -repository is then refused for an hour. When a refresh of one already cached -does not fit, it is served as it is and dropped once no request is reading -it, so the next request clones it again with only its branches and tags; if -that is still too large, it is refused for an hour. A commit fetched by ID -that does not fit is refused, and the repository kept. -Least recently used repositories are evicted after each fetch that takes the -cache past `max_size`, never one a request is reading. A transfer -that receives nothing for 10 minutes is abandoned. Git's automatic garbage -collection runs as part of fetches, and files over 8MB are kept without delta -compression, so one large file does not take a fetch's memory. +evicted as soon as no request was reading it. A clone or fetch that would +pass `max_repo_size` is stopped, and nothing of it is kept. A new repository +is then refused for an hour. When a refresh of one already cached does not +fit, it is served as it is and dropped once no request is reading it, so the +next request clones it again with only its branches and tags; if that is +still too large, it is refused for an hour. A commit fetched by ID that does +not fit is refused, and the repository kept. + +`max_size` counts fetches in progress as well as cached repositories, so the +cache stays within it however many fetches run at once. As a fetch receives +data, the least recently used repositories are evicted to make room, never +one a request is reading; those are evicted once released if the cache is +still over. When evicting every idle repository would not make room, none is +evicted for the fetch, and it is stopped and nothing of it is kept, but the +repository is not refused: a clone answers 507, and a refresh serves the +cached copy, as when the upstream fails. The count is of bytes received, so +index files and garbage collection can take the cache a little past +`max_size` until the fetch ends and the repository is measured. +A transfer that receives nothing for 10 minutes is abandoned. Git's automatic +garbage collection runs as part of fetches, and files over 8MB are kept +without delta compression, so one large file does not take a fetch's memory. Packs for clients are built by at most two `upload-pack` processes per CPU the proxy may use, each with one delta-search thread and 64MB of delta diff --git a/internal/config/git.go b/internal/config/git.go index d3a22d98..860adb71 100644 --- a/internal/config/git.go +++ b/internal/config/git.go @@ -84,8 +84,9 @@ type GitConfig struct { // used by this process only, which locks it. Default: ./cache/git. CacheDir string `json:"cache_dir" yaml:"cache_dir"` - // MaxSize evicts the least recently used repositories once their total - // size passes it. Empty or "0" disables the limit. + // MaxSize bounds the cache, counting fetches in progress; the least + // recently used repositories not in use are evicted to stay within it. + // Empty or "0" disables the limit. MaxSize string `json:"max_size" yaml:"max_size"` // MaxRepoSize is the most one repository may take; a clone or fetch that diff --git a/internal/gitcache/budget.go b/internal/gitcache/budget.go new file mode 100644 index 00000000..9d4135e8 --- /dev/null +++ b/internal/gitcache/budget.go @@ -0,0 +1,126 @@ +package gitcache + +import ( + "context" + "sync/atomic" + "time" +) + +const ( + // maxReserveChunk caps the space a fetch reserves ahead of what it has + // received, so the cache is not locked for every read. Each fetch takes + // at most a sixteenth of MaxSize ahead, so fetches share a small cache. + maxReserveChunk = 4 << 20 + reserveChunks = 16 + // cutWait bounds how long a fetch waits for one that was cut off to + // give back its space. + cutWait = time.Minute +) + +// reservation holds space within MaxSize for the bytes a fetch receives, +// until they are counted in the repository's size. Fetches still running +// therefore count toward MaxSize, so the cache stays within it however many +// run at once. +type reservation struct { + c *Cache + // reserved is written under c.mu; reads that find it large enough skip + // the lock. + reserved atomic.Int64 + // cut, guarded by c.mu, marks a fetch stopped because the cache could + // not make room for it. + cut bool +} + +func (c *Cache) newReservation() *reservation { + return &reservation{c: c} +} + +// cover reserves space for a fetch that has received relayed bytes, +// evicting idle repositories as needed. It returns ErrCacheFull when evicting +// every idle repository would not make room, unless another fetch has just +// been cut off, whose space it waits for: one cut off waits for nothing, so +// two fetches that do not fit together do not both fail. +func (r *reservation) cover(ctx context.Context, relayed int64) error { + c := r.c + if c.opts.MaxSize <= 0 || relayed <= r.reserved.Load() { + return nil + } + var timeout <-chan time.Time + for { + c.mu.Lock() + if r.cut { + c.mu.Unlock() + return ErrCacheFull + } + need := relayed - r.reserved.Load() + removed, total := c.evictLocked(need) + fits := total+need <= c.opts.MaxSize + var freed chan struct{} + switch { + case fits: + chunk := min(maxReserveChunk, c.opts.MaxSize/reserveChunks) + n := min(need+chunk, c.opts.MaxSize-total) + r.reserved.Add(n) + c.reserved += n + case c.cutPending > 0: + freed = c.freed + default: + r.cut = true + c.cutPending++ + } + c.mu.Unlock() + c.removeEvicted(removed) + if fits { + return nil + } + if freed == nil { + return ErrCacheFull + } + if timeout == nil { + timeout = time.After(cutWait) + } + select { + case <-freed: + case <-timeout: + c.mu.Lock() + r.cut = true + c.cutPending++ + c.mu.Unlock() + return ErrCacheFull + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// settle counts what the fetch left in the repository's size and gives back +// its reservation together, so the bytes are never counted twice or not at +// all. +func (r *reservation) settle(rp *repo, size int64) { + r.c.mu.Lock() + rp.size.Store(size) + r.releaseLocked() + r.c.mu.Unlock() +} + +// release gives back the reservation of a fetch that left nothing behind. +// It may follow settle. +func (r *reservation) release() { + r.c.mu.Lock() + r.releaseLocked() + r.c.mu.Unlock() +} + +func (r *reservation) releaseLocked() { + c := r.c + n := r.reserved.Swap(0) + if r.cut { + r.cut = false + c.cutPending-- + } else if n == 0 { + return + } + c.reserved -= n + close(c.freed) + c.freed = make(chan struct{}) +} diff --git a/internal/gitcache/budget_test.go b/internal/gitcache/budget_test.go new file mode 100644 index 00000000..506ef2f2 --- /dev/null +++ b/internal/gitcache/budget_test.go @@ -0,0 +1,218 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "errors" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +// cachedRepo adds a repository of the given size to the cache, still in use +// when busy. +func cachedRepo(t *testing.T, tc *testCache, name string, size int64, busy bool) *repo { + t.Helper() + rp := tc.acquire("up/" + name + ".git") + if err := os.MkdirAll(rp.dir, dirPermissions); err != nil { + t.Fatal(err) + } + rp.size.Store(size) + if !busy { + tc.release(rp) + } + return rp +} + +func (tc *testCache) budget() (reserved int64, cutPending int) { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.reserved, tc.cutPending +} + +func TestCoverEvictsIdleRepositories(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + old := cachedRepo(t, tc, "old", 60, false) + busy := cachedRepo(t, tc, "busy", 30, true) + + res := tc.newReservation() + if err := res.cover(t.Context(), 20); err != nil { + t.Fatal(err) + } + if fileExists(old.dir) { + t.Error("idle repository was not evicted") + } + if !fileExists(busy.dir) { + t.Error("evicted a repository in use") + } + if reserved, _ := tc.budget(); reserved < 20 { + t.Errorf("reserved %d, want at least 20", reserved) + } + res.release() + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after release", reserved) + } +} + +func TestCoverCutsOffOnceOnlyRepositoriesInUseAreLeft(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + busy := cachedRepo(t, tc, "busy", 90, true) + + res := tc.newReservation() + if err := res.cover(t.Context(), 20); !errors.Is(err, ErrCacheFull) { + t.Fatalf("cover: %v, want ErrCacheFull", err) + } + if !fileExists(busy.dir) { + t.Error("evicted a repository in use") + } + res.release() + if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { + t.Errorf("reserved %d, cut off %d after release", reserved, cutPending) + } +} + +// TestCoverKeepsIdleRepositoriesItCannotUse cuts off a fetch that would not +// fit even with every idle repository evicted, so none is. +func TestCoverKeepsIdleRepositoriesItCannotUse(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + idle := cachedRepo(t, tc, "idle", 5, false) + cachedRepo(t, tc, "busy", 90, true) + + res := tc.newReservation() + if err := res.cover(t.Context(), 20); !errors.Is(err, ErrCacheFull) { + t.Fatalf("cover: %v, want ErrCacheFull", err) + } + if !fileExists(idle.dir) { + t.Error("evicted an idle repository without making room") + } + res.release() +} + +// TestCoverWaitsForCutOffFetch runs two fetches that do not fit together. +// Once one is cut off, the other waits for its space instead of failing too. +func TestCoverWaitsForCutOffFetch(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + cachedRepo(t, tc, "busy", 40, true) + + first, second := tc.newReservation(), tc.newReservation() + if err := first.cover(t.Context(), 30); err != nil { + t.Fatal(err) + } + if err := second.cover(t.Context(), 20); err != nil { + t.Fatal(err) + } + if err := first.cover(t.Context(), 40); !errors.Is(err, ErrCacheFull) { + t.Fatalf("first: %v, want ErrCacheFull", err) + } + + done := make(chan error, 1) + go func() { done <- second.cover(context.Background(), 30) }() + select { + case err := <-done: + t.Fatalf("second did not wait for the first to be released: %v", err) + case <-time.After(100 * time.Millisecond): + } + first.release() + if err := <-done; err != nil { + t.Fatalf("second: %v", err) + } + second.release() + if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { + t.Errorf("reserved %d, cut off %d after release", reserved, cutPending) + } +} + +// TestConcurrentClonesStayWithinMaxSize clones two repositories at once that +// do not fit in MaxSize together. Both are kept in use, so neither can be +// evicted: one clone succeeds, and the other is cut off without being +// marked too large. +func TestConcurrentClonesStayWithinMaxSize(t *testing.T) { + upstream := newFakeUpstream(t) + for _, name := range []string{"a", "b"} { + work := upstream.newRepo(name) + big := make([]byte, 256<<10) + _, _ = rand.Read(big) + if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "big.bin") + runGit(t, work, "commit", "--quiet", "-m", "big") + runGit(t, work, "push", "--quiet", "origin", "main") + } + size := repoSize(t, upstream, "a") + maxSize := size * 5 / 3 + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = maxSize }) + for _, name := range []string{"a", "b"} { + defer tc.release(tc.acquire("up/" + name + ".git")) + } + + // Sample what the cache counts while the clones run. + var peak int64 + stop := make(chan struct{}) + sampled := make(chan struct{}) + go func() { + defer close(sampled) + for { + tc.mu.Lock() + total := tc.reserved + for _, rp := range tc.repos { + total += rp.size.Load() + } + tc.mu.Unlock() + peak = max(peak, total) + select { + case <-stop: + return + case <-time.After(100 * time.Microsecond): + } + } + }() + + // Held back, both fetches are under way before either receives a pack. + upstream.delayFetches.Store(int64(200 * time.Millisecond)) + statuses := make(map[string]int) + var mu sync.Mutex + var wg sync.WaitGroup + for _, name := range []string{"a", "b"} { + wg.Go(func() { + resp, err := http.Get(tc.url(name) + "/info/refs?service=git-upload-pack") + if err != nil { + t.Error(err) + return + } + _ = resp.Body.Close() + mu.Lock() + statuses[name] = resp.StatusCode + mu.Unlock() + }) + } + wg.Wait() + close(stop) + <-sampled + + if (statuses["a"] == http.StatusOK) == (statuses["b"] == http.StatusOK) { + t.Fatalf("statuses %v, want one 200 and one 507", statuses) + } + for name, status := range statuses { + if status != http.StatusOK && status != http.StatusInsufficientStorage { + t.Errorf("%s: status %d, want 507", name, status) + } + if tc.isTooLarge("up/" + name + ".git") { + t.Errorf("%s was marked too large", name) + } + } + // What a clone writes beyond the bytes it receives, such as its index, + // is counted only once it is measured. + if slack := size / 4; peak > maxSize+slack { + t.Errorf("cache counted %d, past max_size %d", peak, maxSize) + } + if reserved, cutPending := tc.budget(); reserved != 0 || cutPending != 0 { + t.Errorf("reserved %d, cut off %d once the clones ended", reserved, cutPending) + } + if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { + t.Errorf("left %d entries in the temporary directory", len(entries)) + } +} diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index d65e3df6..4014fd50 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -105,6 +105,13 @@ type Cache struct { // overBudget records that eviction left the cache over MaxSize because // the rest of it was in use. overBudget bool + // reserved is the space reservations hold for fetches in progress, and + // cutPending counts those cut off and not yet released. freed is closed + // and replaced whenever a reservation gives back space or one cut off is + // released. + reserved int64 + cutPending int + freed chan struct{} // failedWants records, by repository key and object ID, wants the // upstream could not provide. failedWants map[string]time.Time @@ -192,6 +199,7 @@ func New(opts Options) (*Cache, error) { repos: make(map[string]*repo), tooLarge: make(map[string]time.Time), failedWants: make(map[string]time.Time), + freed: make(chan struct{}), } if opts.refCheckTimeout > 0 { c.refCheckTimeout = opts.refCheckTimeout @@ -385,21 +393,36 @@ func (rp *repo) exists() bool { // evict removes repositories nobody is using: the least recently used until // the cache fits in MaxSize, and any that could not be refreshed within // MaxRepoSize, so the next request clones them again with only their -// branches and tags. It runs after every fetch, since only fetches grow the -// cache, and when a repository it had to skip is released: one too large to -// refresh, or any while the cache is over MaxSize. A repository is moved -// aside under the lock, so a request that comes after it clones it again -// instead of reading it while it is deleted. +// branches and tags. It runs after every fetch, and when a repository it had +// to skip is released: one too large to refresh, or any while the cache is +// over MaxSize. Fetches also evict as they receive bytes (reservation.cover). func (c *Cache) evict() { c.mu.Lock() - var total int64 + removed, _ := c.evictLocked(0) + c.mu.Unlock() + c.removeEvicted(removed) +} + +// evictLocked evicts until the cache, counting reservations, has room for +// extra more bytes, and returns the evicted repositories' directories, to be +// deleted once c.mu is released, and the size left. When evicting every idle +// repository would not make that room, it evicts only as evict does. A +// repository is moved aside under the lock, so a request that comes after it +// clones it again instead of reading it while it is deleted. +func (c *Cache) evictLocked(extra int64) ([]string, int64) { + total := c.reserved + var idleTotal int64 idle := make([]*repo, 0, len(c.repos)) for _, rp := range c.repos { total += rp.size.Load() if rp.users == 0 { idle = append(idle, rp) + idleTotal += rp.size.Load() } } + if total-idleTotal+extra > c.opts.MaxSize { + extra = 0 + } slices.SortFunc(idle, func(a, b *repo) int { return cmp.Compare(a.lastUsed.Load(), b.lastUsed.Load()) }) @@ -407,7 +430,7 @@ func (c *Cache) evict() { for _, rp := range idle { _, marked := c.tooLarge[rp.key] tooLarge := marked && rp.exists() - if !tooLarge && (c.opts.MaxSize <= 0 || total <= c.opts.MaxSize) { + if !tooLarge && (c.opts.MaxSize <= 0 || total+extra <= c.opts.MaxSize) { continue } trash, err := os.MkdirTemp(c.tmpDir(), "evicted-") @@ -431,8 +454,10 @@ func (c *Cache) evict() { c.logger.Info("git cache: evicted repository", "repository", rp.key, "size", rp.size.Load(), "reason", reason) } c.overBudget = c.opts.MaxSize > 0 && total > c.opts.MaxSize - c.mu.Unlock() + return removed, total +} +func (c *Cache) removeEvicted(removed []string) { for _, trash := range removed { if err := os.RemoveAll(trash); err != nil { c.logger.Warn("git cache: failed to delete evicted repository", "path", trash, "error", err) diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 9f558813..e2c4a27e 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -318,7 +318,7 @@ func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { case errors.Is(err, ErrNotFound): http.Error(w, err.Error(), http.StatusNotFound) return - case errors.Is(err, ErrTooLarge): + case errors.Is(err, ErrTooLarge), errors.Is(err, ErrCacheFull): http.Error(w, err.Error(), http.StatusInsufficientStorage) return case err != nil: diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index dd9af959..694ea123 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -185,6 +185,9 @@ func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked f // clone builds the repository in a temporary directory and moves it into // place, so no request ever reads a partial clone. func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { + // Released after the temporary clone is removed. + res := c.newReservation() + defer res.release() tmp, err := os.MkdirTemp(c.tmpDir(), "clone-") if err != nil { return err @@ -194,7 +197,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { return err } - if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, refreshFlags, branchesAndTags...); err != nil { + if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil { return err } if err := c.setHead(ctx, tmp, remote, ""); err != nil { @@ -210,7 +213,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo if err := os.Rename(tmp, rp.dir); err != nil { return fmt.Errorf("moving clone into place: %w", err) } - rp.size.Store(size) + res.settle(rp, size) c.logger.Info("git cache: cloned repository", "repository", rp.key, "size", size) return nil } @@ -234,8 +237,9 @@ func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote rem if err != nil { return err } - err = c.fetch(ctx, rp.dir, repoURL, limit, refreshFlags, branchesAndTags...) - rp.size.Store(dirSize(rp.dir)) + res := c.newReservation() + err = c.fetch(ctx, rp.dir, repoURL, limit, res, refreshFlags, branchesAndTags...) + res.settle(rp, dirSize(rp.dir)) if err != nil { return err } @@ -271,8 +275,8 @@ func (c *Cache) fetchLimit(rp *repo) (int64, error) { // fetch fetches refspecs from the upstream into the repository at dir, // through the relay, adding flags to git fetch. limit caps the bytes -// fetched, 0 for no cap. -func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, flags []string, refspecs ...string) error { +// fetched, 0 for no cap, and res holds space within MaxSize for them. +func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res *reservation, flags []string, refspecs ...string) error { select { case c.fetches <- struct{}{}: defer func() { <-c.fetches }() @@ -280,7 +284,7 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, fla return ctx.Err() } - relayURL, route, unregister := c.relay.register(repoURL, limit) + relayURL, route, unregister := c.relay.register(repoURL, limit, res) defer unregister() args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) @@ -296,6 +300,9 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, fla if route.exceeded.Load() { return ErrTooLarge } + if route.full.Load() { + return ErrCacheFull + } if err != nil && !route.answered.Load() { return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) } @@ -345,6 +352,8 @@ func errorType(err error) string { return "refused" case errors.Is(err, ErrTooLarge): return "too_large" + case errors.Is(err, ErrCacheFull): + return "cache_full" } return "fetch_failed" } diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index 437d7b55..38baa295 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -39,10 +39,14 @@ type relay struct { type relayRoute struct { repoURL string - // limit caps the response bytes the route relays, 0 for no cap. + // limit caps the response bytes the route relays, 0 for no cap, and + // budget, when set, holds space within MaxSize for them. exceeded and + // full record which of them stopped the route. limit int64 + budget *reservation relayed atomic.Int64 exceeded atomic.Bool + full atomic.Bool // answered records that the upstream answered a git-upload-pack // request, so a failed fetch was not an outage. answered atomic.Bool @@ -77,11 +81,11 @@ func (rl *relay) close() error { // register adds a route to repoURL and returns the URL git fetches from and // a function that removes the route. -func (rl *relay) register(repoURL string, limit int64) (string, *relayRoute, func()) { +func (rl *relay) register(repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { var b [relayIDBytes]byte _, _ = rand.Read(b[:]) id := hex.EncodeToString(b[:]) - route := &relayRoute{repoURL: repoURL, limit: limit} + route := &relayRoute{repoURL: repoURL, limit: limit, budget: budget} rl.mu.Lock() rl.routes[id] = route @@ -147,8 +151,8 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", ct) } w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, &routeReader{r: resp.Body, route: route}); err != nil { - if route.exceeded.Load() { + if _, err := io.Copy(w, &routeReader{ctx: r.Context(), r: resp.Body, route: route}); err != nil { + if route.exceeded.Load() || route.full.Load() { // Cut the connection so git fails rather than keeping a pack // that merely looks short. panic(http.ErrAbortHandler) @@ -159,17 +163,27 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -// routeReader counts relayed bytes against the route's limit. +// routeReader counts relayed bytes against the route's limit and budget. type routeReader struct { + ctx context.Context r io.Reader route *relayRoute } func (rr *routeReader) Read(p []byte) (int, error) { n, err := rr.r.Read(p) - if total := rr.route.relayed.Add(int64(n)); rr.route.limit > 0 && total > rr.route.limit { + total := rr.route.relayed.Add(int64(n)) + if rr.route.limit > 0 && total > rr.route.limit { rr.route.exceeded.Store(true) return 0, ErrTooLarge } + if rr.route.budget != nil { + if err := rr.route.budget.cover(rr.ctx, total); err != nil { + if errors.Is(err, ErrCacheFull) { + rr.route.full.Store(true) + } + return 0, err + } + } return n, err } diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go index 45726bcb..4ee9142a 100644 --- a/internal/gitcache/relay_test.go +++ b/internal/gitcache/relay_test.go @@ -27,7 +27,7 @@ func TestRelayOnlyForwardsFetchRequests(t *testing.T) { })) defer upstream.Close() rl := newTestRelay(t) - base, _, unregister := rl.register(upstream.URL+"/repo", 0) + base, _, unregister := rl.register(upstream.URL+"/repo", 0, nil) refused := []struct{ method, url string }{ {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, @@ -71,7 +71,7 @@ func TestRelayStopsAtLimit(t *testing.T) { })) defer upstream.Close() rl := newTestRelay(t) - base, route, unregister := rl.register(upstream.URL+"/repo", limit) + base, route, unregister := rl.register(upstream.URL+"/repo", limit, nil) defer unregister() resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) diff --git a/internal/gitcache/upstream.go b/internal/gitcache/upstream.go index 83c14349..46a2667c 100644 --- a/internal/gitcache/upstream.go +++ b/internal/gitcache/upstream.go @@ -33,6 +33,9 @@ var ( // ErrTooLarge is returned for a repository larger than git.max_repo_size, // or than git.max_size where that is smaller. ErrTooLarge = errors.New("repository is larger than git.max_repo_size or git.max_size") + // ErrCacheFull is returned for a fetch cut off because repositories in + // use take what git.max_size leaves. + ErrCacheFull = errors.New("git cache is full") ) // remoteRefs is what a repository advertises: its branch and tag tips, and diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go index eec8fc45..c715f2f3 100644 --- a/internal/gitcache/wants.go +++ b/internal/gitcache/wants.go @@ -93,10 +93,10 @@ func (c *Cache) fetchMissingWants(ctx context.Context, rp *repo, repoURL string, return err } if len(missing) == 1 || len(missing) > maxWantSplit || - errors.Is(err, ErrTooLarge) || errors.Is(err, errUpstreamUnanswered) { + errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { // Fetched one at a time, wants too large for what max_repo_size - // leaves would exhaust the same budget, and an upstream that did - // not answer would be waited on once per want. + // or max_size leaves would exhaust the same budget, and an upstream + // that did not answer would be waited on once per want. c.recordFailedWants(rp.key, missing, err) return err } @@ -130,8 +130,9 @@ func (c *Cache) fetchWants(ctx context.Context, rp *repo, repoURL string, oids [ for i, oid := range oids { refspecs[i] = oid + ":" + wantedRef(oid, now) } - err = c.fetch(ctx, rp.dir, repoURL, limit, []string{"--atomic"}, refspecs...) - rp.size.Store(dirSize(rp.dir)) + res := c.newReservation() + err = c.fetch(ctx, rp.dir, repoURL, limit, res, []string{"--atomic"}, refspecs...) + res.settle(rp, dirSize(rp.dir)) return err } diff --git a/internal/handler/git.go b/internal/handler/git.go index d7d8dd04..3fbacfd7 100644 --- a/internal/handler/git.go +++ b/internal/handler/git.go @@ -207,6 +207,8 @@ func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err erro http.Error(w, "upstream refused the request", http.StatusBadGateway) case errors.Is(err, gitcache.ErrTooLarge): http.Error(w, "repository is larger than git.max_repo_size or git.max_size", http.StatusInsufficientStorage) + case errors.Is(err, gitcache.ErrCacheFull): + http.Error(w, "git cache is full", http.StatusInsufficientStorage) default: http.Error(w, "upstream unavailable", http.StatusBadGateway) }