diff --git a/README.md b/README.md index f3508dd0..f7d8c309 100644 --- a/README.md +++ b/README.md @@ -1014,6 +1014,8 @@ proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. +With `scanning.enabled`, the mirror command refuses every package and container image so nothing is cached unscanned. Each one is reported as failed, so the command exits with status 1. Scanners fetch the staged bytes from the running server, which this command does not start. Use the mirror API below instead, since its jobs run inside the server and go through the pre-cache scan. `--dry-run` still lists what would be mirrored. + A mirror API is also available when the server is running: ```bash diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 0b140ea3..bfc07747 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -558,9 +558,7 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images if cfg.Scanning.Enabled { - // Scanners fetch staged layers from the running server, which this - // command does not start. - m.Images = mirror.UnavailableImages{Reason: "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"} + refuseUnscannedMirroring(m) } ctx, cancel := context.WithCancel(context.Background()) @@ -617,6 +615,23 @@ func mirrorExitCode(progress *mirror.Progress) int { return 0 } +// Reasons the mirror command gives for refusing items while scanning is +// enabled. +const ( + mirrorScanningImagesReason = "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server" + mirrorScanningPackagesReason = "packages cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server" +) + +// refuseUnscannedMirroring makes m fail every item instead of caching it. +// Scanners fetch the staged bytes from the running server, which this command +// does not start, so anything it cached would skip the pre-cache scan and +// later be served from the cache unscanned. The mirror API's jobs run inside +// the server with its scanners and are unaffected. +func refuseUnscannedMirroring(m *mirror.Mirror) { + m.Images = mirror.UnavailableImages{Reason: mirrorScanningImagesReason} + m.RefusePackages = mirrorScanningPackagesReason +} + func printStats(db *database.DB, popular, recent int, asJSON bool) error { defer func() { _ = db.Close() }() diff --git a/cmd/proxy/mirror_test.go b/cmd/proxy/mirror_test.go index 4db2a898..e0f7e6af 100644 --- a/cmd/proxy/mirror_test.go +++ b/cmd/proxy/mirror_test.go @@ -8,9 +8,133 @@ import ( "strings" "testing" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/mirror" ) +// mirrorTestEnv carries the mirror command's arguments into the helper +// process, one per line. +const mirrorTestEnv = "PROXY_TEST_MIRROR_ARGS" + +// writeScanningMirrorConfig writes a config with scanning enabled and storage +// and database under dir. The scanner address is never reached. +func writeScanningMirrorConfig(t *testing.T, dir string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "scanning:\n" + + " enabled: true\n" + + " signing_key: test-signing-key\n" + + " scanners:\n" + + " - name: test\n" + + " url: http://127.0.0.1:1/scan\n" + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +// runMirrorProcess runs proxy mirror with args in a helper process and +// checks that it exits with wantCode. +func runMirrorProcess(t *testing.T, wantCode int, args ...string) string { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorHelperProcess$") + cmd.Env = append(os.Environ(), mirrorTestEnv+"="+strings.Join(args, "\n")) + output, err := cmd.CombinedOutput() + + code := 0 + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("proxy mirror %v: %v\n%s", args, err, output) + } + if code != wantCode { + t.Fatalf("proxy mirror %v exited with status %d, want %d\n%s", args, code, wantCode, output) + } + return string(output) +} + +// With scanning enabled, proxy mirror refuses packages rather than caching +// them unscanned: scanners fetch staged bytes from the running server, which +// the command does not start. +// A refused package counts as failed, so the command exits 1. +func TestMirrorCommandRefusesPackagesWhileScanning(t *testing.T) { + dir := t.TempDir() + configPath := writeScanningMirrorConfig(t, dir) + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:npm/lodash@4.17.21") + + for _, want := range []string{mirrorScanningPackagesReason, "0 downloaded", "1 failed"} { + if !strings.Contains(output, want) { + t.Errorf("output missing %q:\n%s", want, output) + } + } + + db, err := database.Open(filepath.Join(dir, "proxy.db")) + if err != nil { + t.Fatalf("opening database: %v", err) + } + defer func() { _ = db.Close() }() + stats, err := db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + if stats.TotalArtifacts != 0 { + t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts) + } + + var stored []string + _ = filepath.WalkDir(filepath.Join(dir, "artifacts"), func(path string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + stored = append(stored, path) + } + return nil + }) + if len(stored) != 0 { + t.Errorf("files written to storage: %v", stored) + } +} + +// A dry run caches nothing, so it still lists what would be mirrored. +func TestMirrorCommandDryRunWhileScanning(t *testing.T) { + configPath := writeScanningMirrorConfig(t, t.TempDir()) + + output := runMirrorProcess(t, 0, "-config", configPath, "-dry-run", "pkg:npm/lodash@4.17.21") + + if !strings.Contains(output, "Would mirror 1 package versions") { + t.Errorf("dry run output:\n%s", output) + } + if strings.Contains(output, mirrorScanningPackagesReason) { + t.Errorf("dry run reported the refusal:\n%s", output) + } +} + +func TestRefuseUnscannedMirroring(t *testing.T) { + m := mirror.New(nil, nil, nil, nil, 1) + refuseUnscannedMirroring(m) + + if m.RefusePackages != mirrorScanningPackagesReason { + t.Errorf("RefusePackages = %q, want %q", m.RefusePackages, mirrorScanningPackagesReason) + } + images, ok := m.Images.(mirror.UnavailableImages) + if !ok || images.Reason != mirrorScanningImagesReason { + t.Errorf("Images = %#v, want UnavailableImages with the scanning reason", m.Images) + } +} + +func TestMirrorHelperProcess(*testing.T) { + args := os.Getenv(mirrorTestEnv) + if args == "" { + return + } + os.Args = append([]string{"proxy", "mirror"}, strings.Split(args, "\n")...) + main() +} + func TestMirrorExitCode(t *testing.T) { tests := []struct { name string diff --git a/docs/configuration.md b/docs/configuration.md index 5fcdc237..607d8a60 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -862,6 +862,8 @@ Scanners configured for the same ecosystem all run concurrently, never sequentia A blocked download surfaces to the client as `403 Forbidden` with the scanner's reason, across every ecosystem handler. +Because scanners pull the staged bytes from the running server, the `proxy mirror` command cannot scan what it fetches. While scanning is enabled it refuses every package and container image instead of caching them unscanned. Mirror through the `/api/mirror` endpoints of the running server, whose jobs are scanned like any other download. + ### Scanner HTTP contract Any external service that implements this contract can act as a scanner — a trivy wrapper, a clamav-rest bridge, a Wiz connector, or an in-house service. The proxy POSTs a notify request to `scanning.scanners[].url` and waits for a JSON verdict. diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index fb98c11f..e9468253 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -26,6 +26,10 @@ type Mirror struct { // Images mirrors docker and oci items. Without it they fail like any // other unsupported ecosystem. Images ImageMirrorer + + // RefusePackages, when set, fails every item Images does not take with + // this reason instead of fetching it, so nothing is cached. + RefusePackages string } // New creates a new Mirror with the given dependencies. @@ -211,6 +215,13 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorImage(ctx, pv, tracker) return } + if m.RefusePackages != "" { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, m.RefusePackages) + m.logger.Warn("mirror refused", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { tracker.failed.Add(1) diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go index 45dd6f38..68526878 100644 --- a/internal/mirror/mirror_test.go +++ b/internal/mirror/mirror_test.go @@ -3,10 +3,12 @@ package mirror import ( "context" "database/sql" + "errors" "log/slog" "net/http" "os" "strings" + "sync" "testing" "time" @@ -261,3 +263,90 @@ func TestNewMirrorDefaultWorkers(t *testing.T) { t.Errorf("workers = %d, want 1 (minimum)", m.workers) } } + +// recordingFetcher fails every fetch and records the URLs it was asked for, so +// a test can show nothing reached upstream. +type recordingFetcher struct { + mu sync.Mutex + urls []string +} + +func (f *recordingFetcher) record(url string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.urls = append(f.urls, url) + return errors.New("recordingFetcher: no upstream in this test") +} + +func (f *recordingFetcher) Fetch(_ context.Context, url string) (*fetch.Artifact, error) { + return nil, f.record(url) +} + +func (f *recordingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { + return nil, f.record(url) +} + +func (f *recordingFetcher) Head(_ context.Context, url string) (int64, string, error) { + return 0, "", f.record(url) +} + +func (f *recordingFetcher) fetched() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.urls...) +} + +func TestMirrorRunRefusesPackages(t *testing.T) { + m := setupTestMirror(t, 2) + fetcher := &recordingFetcher{} + m.proxy.Fetcher = fetcher + const reason = "packages cannot be mirrored here" + m.RefusePackages = reason + + source := &PURLSource{PURLs: []string{"pkg:npm/lodash@4.17.21", "pkg:cargo/serde@1.0.0"}} + progress, err := m.Run(context.Background(), source) + if err != nil { + t.Fatalf("Run: %v", err) + } + + if progress.Failed != 2 || progress.Completed != 0 || progress.Skipped != 0 { + t.Errorf("progress = %d failed, %d completed, %d skipped; want 2 failed only", + progress.Failed, progress.Completed, progress.Skipped) + } + if len(progress.Errors) != 2 { + t.Fatalf("errors = %+v, want one per package", progress.Errors) + } + for _, e := range progress.Errors { + if e.Error != reason { + t.Errorf("%s/%s error = %q, want %q", e.Ecosystem, e.Name, e.Error, reason) + } + } + if urls := fetcher.fetched(); len(urls) != 0 { + t.Errorf("refused packages were fetched from upstream: %v", urls) + } + + stats, err := m.db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + if stats.TotalArtifacts != 0 { + t.Errorf("cached artifacts = %d, want 0", stats.TotalArtifacts) + } +} + +// Images keep their own refusal, so a mirror refusing packages still hands +// docker and oci items to Images. +func TestMirrorRefusePackagesLeavesImagesToImages(t *testing.T) { + m := setupTestMirror(t, 1) + m.proxy.Fetcher = &recordingFetcher{} + m.RefusePackages = "packages refused" + m.Images = UnavailableImages{Reason: "images refused"} + + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:docker/library/alpine@3.20"}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(progress.Errors) != 1 || progress.Errors[0].Error != "images refused" { + t.Errorf("errors = %+v, want the image refusal", progress.Errors) + } +}