From 9b7ea47a3741ff23a3b83b282f5b42ff24a2b9a3 Mon Sep 17 00:00:00 2001 From: manjari Date: Thu, 22 Oct 2026 04:31:18 +0000 Subject: [PATCH] feat(demo): Add an ANALYZER=tango option to the local demo --- Makefile | 24 +++++-- doc/howto/QUICKSTART.md | 24 +++++++ service/submitqueue/demo/provider/README.md | 2 + .../submitqueue/demo/provider/git/BUILD.bazel | 8 +++ .../demo/provider/git/profiles.tango.yaml | 43 ++++++++++++ service/submitqueue/demo/requests/BUILD.bazel | 1 + service/submitqueue/demo/requests/main.go | 22 ++++++- .../submitqueue/demo/requests/source_test.go | 38 +++++++++++ service/submitqueue/demo/workload.go | 5 +- service/submitqueue/demo/workload_test.go | 2 +- service/submitqueue/docker-compose.tango.yml | 28 ++++++++ .../orchestrator/server/BUILD.bazel | 5 +- .../orchestrator/server/config_test.go | 6 ++ tool/gitsandbox/main.go | 65 ++++++++++++++++--- tool/gitsandbox/main_test.go | 38 ++++++++++- 15 files changed, 289 insertions(+), 22 deletions(-) create mode 100644 service/submitqueue/demo/provider/git/profiles.tango.yaml create mode 100644 service/submitqueue/docker-compose.tango.yml diff --git a/Makefile b/Makefile index 8f47af53d..f1332667a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,13 @@ export SQ_PROVIDER_CONFIG_DIR ?= $(REPO_ROOT)/service/submitqueue/demo/provider/ PROVIDER_COMPOSE_FILE_fake = $(abspath service/submitqueue/docker-compose.fake.yml) PROVIDER_COMPOSE_FILE_git = $(abspath service/submitqueue/docker-compose.git.yml) PROVIDER_COMPOSE_FILE_github = $(abspath service/submitqueue/docker-compose.provider.yml) -PROVIDER_COMPOSE_FILE = $(PROVIDER_COMPOSE_FILE_$(PROVIDER)) +# What answers conflict analysis, chosen independently of the provider. Empty +# keeps the provider's own analyzer; tango adds a fake Tango server and needs +# PROVIDER=git, whose repository it reads targets from. +ANALYZER ?= +ANALYZER_COMPOSE_FILE_tango = $(abspath service/submitqueue/docker-compose.tango.yml) +ANALYZER_COMPOSE_FILE = $(ANALYZER_COMPOSE_FILE_$(ANALYZER)) +PROVIDER_COMPOSE_FILE = $(PROVIDER_COMPOSE_FILE_$(PROVIDER))$(if $(ANALYZER_COMPOSE_FILE), -f $(ANALYZER_COMPOSE_FILE)) # Where PROVIDER=git keeps the bare repository it lands into. Outside the # repository, so a demo leaves nothing in a checkout, and bind-mounted rather @@ -165,7 +171,7 @@ build: ## Build all services and examples # Build Linux binaries required for the local docker-compose flows (local-*). # Bazel-run integration/e2e tests do NOT need these: they build the service # binaries hermetically as data dependencies. -build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-linux ## Build all Linux binaries for Docker +build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-linux build-tangofake-linux ## Build all Linux binaries for Docker @echo "All Linux binaries ready for Docker" build-runway-linux: ## Build Runway Linux binary for Docker @@ -389,7 +395,7 @@ local-submitqueue-clean: ## Stop the stack and remove its volumes, images, and P @echo "Cleaning all services and data..." @# The overlay is named so that volumes it declares — Runway's checkouts — @# are removed too, rather than surviving as an orphan. - @$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down -v --rmi local + @$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down --remove-orphans -v --rmi local @rm -rf "$(SQ_GIT_SANDBOX_DIR)" @echo "All services, volumes, images, and $(SQ_GIT_SANDBOX_DIR) removed." @@ -534,14 +540,18 @@ local-submitqueue-restart: build-all-linux ## Restart all services (rebuild and @echo "Services restarted!" @make local-submitqueue-ps -local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PROVIDER=fake|git|github; github needs GITHUB_TOKEN) +local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PROVIDER=fake|git|github; github needs GITHUB_TOKEN; ANALYZER=tango with PROVIDER=git adds a fake Tango server) + @test -z "$(ANALYZER)" || test -n "$(ANALYZER_COMPOSE_FILE)" \ + || { echo "Unknown ANALYZER '$(ANALYZER)'; the only choice is tango"; exit 2; } + @test -z "$(ANALYZER)" || test "$(PROVIDER)" = "git" \ + || { echo "ANALYZER=$(ANALYZER) needs PROVIDER=git, got '$(PROVIDER)'"; exit 2; } @echo "Starting full stack against provider '$(PROVIDER)' ($(SQ_PROVIDER_CONFIG_DIR))..." @test -f "$(SQ_PROVIDER_CONFIG_DIR)/merge.yaml" \ || { echo "No such provider '$(PROVIDER)': $(SQ_PROVIDER_CONFIG_DIR)/merge.yaml not found"; exit 2; } @test -n "$(PROVIDER_COMPOSE_FILE)" \ || { echo "Provider '$(PROVIDER)' has no compose overlay; add PROVIDER_COMPOSE_FILE_$(PROVIDER) to the Makefile"; exit 2; } @if [ "$(PROVIDER)" = "git" ]; then \ - $(BAZEL) run //tool/gitsandbox -- -sandbox-dir "$(SQ_GIT_SANDBOX_DIR)" || exit 1; \ + $(BAZEL) run //tool/gitsandbox -- -sandbox-dir "$(SQ_GIT_SANDBOX_DIR)" $(if $(ANALYZER),-targets) || exit 1; \ fi @# Rootless Docker maps container root to the host user; rootful Docker needs @# the host UID:GID explicitly, or the services write files into the sandbox @@ -572,7 +582,7 @@ local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PRO local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sandbox; the databases do not survive) @echo "Stopping SubmitQueue services..." - @$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down + @$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down --remove-orphans @# Both MySQL services mount anonymous volumes, so `down` detaches them and @# the next `up` creates fresh ones. Saying "data preserved" here would be @# read as "your requests are still there", which they are not. @@ -584,7 +594,7 @@ local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sand local-stop: ## Stop every local stack — SubmitQueue, Stovepipe, and Runway @echo "Stopping all services..." - @$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down + @$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down --remove-orphans @$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -f $(STOVEPIPE_WEB_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down @$(COMPOSE) -f $(RUNWAY_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down @echo "Services stopped. Anonymous database volumes are not reused on the next start." diff --git a/doc/howto/QUICKSTART.md b/doc/howto/QUICKSTART.md index ea6e30e0f..aef709ebd 100644 --- a/doc/howto/QUICKSTART.md +++ b/doc/howto/QUICKSTART.md @@ -259,6 +259,30 @@ Three changes, one more ref update than before. The sandbox survives `make local-submitqueue-stop`, so what landed is still there to look at. `make local-submitqueue-clean` removes it. +## Decide conflicts with Tango + +`PROVIDER=git` decides that two changes conflict by comparing the directories they touch. An analyzer built on a build graph can see more: a change to a library conflicts with a change to anything built on it, though their paths never overlap. `ANALYZER=tango` swaps the analyzer for one that asks a Tango server which build targets each batch affects, and runs a fake Tango server beside the stack so it needs no build system. + +```bash +make local-submitqueue-stop +make local-submitqueue-start PROVIDER=git ANALYZER=tango +make demo-requests +make local-submitqueue-stop PROVIDER=git ANALYZER=tango +``` + +Starting with `ANALYZER=tango` also commits a `TARGET` file into each `demo/NN` folder of the sandbox, on top of whatever is already there. `demo/00` and `demo/01` are libraries, and every other folder depends on one of them: even folders on `demo/00`, odd folders on `demo/01`. Pass the same `ANALYZER` when stopping, so the fake server is stopped with the rest. Only the folders `demo/00` to `demo/09` have targets, so `make demo-requests` refuses a `FOLDERS` above 10 against a sandbox seeded this way, because changes in a folder without a target would never conflict. + +Look at what the analyzer recorded, the same way as in the speculation section above: + +```bash +docker exec submitqueue-mysql-app-1 mysql -uroot -proot submitqueue \ + -N -e "select batch_id, dependents from batch_dependent where dependents != '[]'" +docker exec submitqueue-mysql-app-1 mysql -uroot -proot submitqueue \ + -N -e "select count(*) from target_signature" +``` + +The second count has a row for every batch analyzed, which shows the orchestrator really asked the fake server. Dependencies between batches can now join different folders, where under path overlap they only join batches sharing one. + ## Land against GitHub The last rung is the only one that needs credentials, and the only one where a change is a pull request that a person can review. diff --git a/service/submitqueue/demo/provider/README.md b/service/submitqueue/demo/provider/README.md index cbfab70b8..7a9c4496c 100644 --- a/service/submitqueue/demo/provider/README.md +++ b/service/submitqueue/demo/provider/README.md @@ -19,6 +19,8 @@ Pick one with `make local-submitqueue-start PROVIDER=`, which bind-mounts The three are a ladder, and the rung is the only thing that changes: the same commands land against all of them. `git/` is worth reading first of the two real ones. It is proof that neither half needs a provider — change metadata comes from reading the repository and the merge is the same Runway code path, both against a bare repository addressed by path, with no credential and no API — and it is what the hermetic git E2E (`make e2e-git-test`) runs against. +The conflict analyzer is a separate choice from the provider. `make local-submitqueue-start PROVIDER=git ANALYZER=tango` keeps everything about `git/` and swaps `demo-queue`'s analyzer for Tango, answered by the fake server in [`service/tangofake`](../../../tangofake). It reads [`git/profiles.tango.yaml`](git/profiles.tango.yaml) instead of `profiles.yaml`, and the sandbox gains a `TARGET` file per `demo/NN` folder. Only `PROVIDER=git` supports it, because the fake reads targets from that provider's repository. + Note that the orchestrator and Runway each keep their **own** copy of a queue's repository and configure their own remote for it: the change provider's copy is in `profiles.yaml`, the merger's checkout in `merge.yaml`, and they are independent even when they name the same remote. That is the same shape a deployment has when both point at a remote host. ## Adding a provider diff --git a/service/submitqueue/demo/provider/git/BUILD.bazel b/service/submitqueue/demo/provider/git/BUILD.bazel index 0cd7d9377..4285fb4f1 100644 --- a/service/submitqueue/demo/provider/git/BUILD.bazel +++ b/service/submitqueue/demo/provider/git/BUILD.bazel @@ -8,7 +8,15 @@ filegroup( name = "config", srcs = [ "merge.yaml", + "profiles.tango.yaml", "profiles.yaml", ], visibility = ["//test:__subpackages__"], ) + +# Read by the orchestrator's config test, so the profile `ANALYZER=tango` +# mounts is checked against the loader that will read it. +exports_files( + ["profiles.tango.yaml"], + visibility = ["//service/submitqueue/orchestrator/server:__pkg__"], +) diff --git a/service/submitqueue/demo/provider/git/profiles.tango.yaml b/service/submitqueue/demo/provider/git/profiles.tango.yaml new file mode 100644 index 000000000..9c3aa0cbb --- /dev/null +++ b/service/submitqueue/demo/provider/git/profiles.tango.yaml @@ -0,0 +1,43 @@ +# Extension profiles for the "git" example with the Tango conflict analyzer: +# `make local-submitqueue-start PROVIDER=git ANALYZER=tango`. +# +# The same repository, change provider and build runner as profiles.yaml; the +# only difference is how demo-queue decides that two batches conflict. Instead +# of comparing paths, it asks a Tango server which build targets each batch +# affects, so batches conflict when their targets overlap. The server here is the +# fake in service/tangofake, which reads the TARGET files that tool/gitsandbox +# commits into the sandbox. + +tango: + address: tangofake-service:8080 + # Shared by every queue's analyzer. + signatureCacheBytes: 67108864 + +defaults: + changeProvider: {type: fake} + buildRunner: {type: fake} + analyzer: {type: all} + # Same example factors as ../fake/profiles.yaml. + scorer: + type: evidence + factors: + pathPassed: 10 + pathFailed: 0.3 + landing: 12 + cancelling: 0.1 + base: + type: heuristic + +queues: + - name: demo-queue + changeProvider: + type: git + git: + remoteUrl: file:///srv/git/sandbox.git + target: main + repoPath: /var/submitqueue/changerepos/demo + analyzer: + type: tango + tango: + # The fake serves one repository and ignores the remote. + remote: sandbox diff --git a/service/submitqueue/demo/requests/BUILD.bazel b/service/submitqueue/demo/requests/BUILD.bazel index af577edd3..98f0ba174 100644 --- a/service/submitqueue/demo/requests/BUILD.bazel +++ b/service/submitqueue/demo/requests/BUILD.bazel @@ -47,6 +47,7 @@ go_test( "//platform/base/change/git:go_default_library", "//platform/git/exec:go_default_library", "//platform/git/exectest:go_default_library", + "//service/submitqueue/demo:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", ], diff --git a/service/submitqueue/demo/requests/main.go b/service/submitqueue/demo/requests/main.go index 14b12405a..76795bc75 100644 --- a/service/submitqueue/demo/requests/main.go +++ b/service/submitqueue/demo/requests/main.go @@ -221,7 +221,11 @@ func newChangeSource(ctx context.Context, cfg config) (changeSource, func(), err if err != nil { return nil, nil, err } - src, err := newGitSource(ctx, git, filepath.Join(cfg.sandboxDir, sandboxRepo+".git"), sandboxRepo) + bare := filepath.Join(cfg.sandboxDir, sandboxRepo+".git") + if err := checkFoldersHaveTargets(ctx, git, bare, cfg.base, cfg.folders); err != nil { + return nil, nil, err + } + src, err := newGitSource(ctx, git, bare, sandboxRepo) if err != nil { return nil, nil, err } @@ -235,6 +239,22 @@ func newChangeSource(ctx context.Context, cfg config) (changeSource, func(), err return nil, nil, fmt.Errorf("unknown provider %q", cfg.provider) } +// targetsMarker is the file tool/gitsandbox -targets commits first; it is on +// the base branch only when the stack runs the Tango analyzer. +const targetsMarker = "demo/00/TARGET" + +// checkFoldersHaveTargets rejects a folder count the sandbox's targets do not +// cover, since changes in a folder without a target never conflict. +func checkFoldersHaveTargets(ctx context.Context, git, bare, base string, folders int) error { + if folders <= demo.MaxShardDirs { + return nil + } + if err := gitexec.Run(ctx, git, bare, "cat-file", "-e", base+":"+targetsMarker); err != nil { + return nil + } + return fmt.Errorf("-folders %d is above %d, the folders with targets in this sandbox", folders, demo.MaxShardDirs) +} + // sandboxRepo is the repository the git provider's sandbox holds, matching what // tool/gitsandbox creates and what demo/provider/git/merge.yaml merges into. const sandboxRepo = "sandbox" diff --git a/service/submitqueue/demo/requests/source_test.go b/service/submitqueue/demo/requests/source_test.go index f2a38380d..f4d40963c 100644 --- a/service/submitqueue/demo/requests/source_test.go +++ b/service/submitqueue/demo/requests/source_test.go @@ -28,6 +28,7 @@ import ( gitchange "github.com/uber/submitqueue/platform/base/change/git" gitexec "github.com/uber/submitqueue/platform/git/exec" gitexectest "github.com/uber/submitqueue/platform/git/exectest" + "github.com/uber/submitqueue/service/submitqueue/demo" ) // quietSpec is a changeSpec whose progress notes go nowhere, for tests that @@ -245,3 +246,40 @@ func TestNewChangeSource_GitRejectsAMissingSandbox(t *testing.T) { }) require.Error(t, err, "a missing sandbox must say so rather than fail later mid-run") } + +func TestCheckFoldersHaveTargets(t *testing.T) { + git := gitexectest.Git(t) + ctx := context.Background() + plain := sandbox(t, git) + withTargets := sandbox(t, git) + seed := t.TempDir() + require.NoError(t, gitexec.Run(ctx, git, "", "clone", withTargets, seed)) + require.NoError(t, gitexec.Run(ctx, git, seed, "config", "user.name", "Test")) + require.NoError(t, gitexec.Run(ctx, git, seed, "config", "user.email", "test@example.invalid")) + require.NoError(t, os.MkdirAll(filepath.Join(seed, "demo", "00"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(seed, targetsMarker), nil, 0o644)) + require.NoError(t, gitexec.Run(ctx, git, seed, "add", ".")) + require.NoError(t, gitexec.Run(ctx, git, seed, "commit", "-m", "targets")) + require.NoError(t, gitexec.Run(ctx, git, seed, "push", "origin", "main")) + + tests := []struct { + name string + bare string + folders int + wantErr bool + }{ + {name: "targets cover the folders", bare: withTargets, folders: demo.MaxShardDirs}, + {name: "targets do not cover the folders", bare: withTargets, folders: demo.MaxShardDirs + 1, wantErr: true}, + {name: "no targets, any folder count", bare: plain, folders: 50}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkFoldersHaveTargets(ctx, git, tt.bare, "main", tt.folders) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} diff --git a/service/submitqueue/demo/workload.go b/service/submitqueue/demo/workload.go index e49b00fdb..4586d6544 100644 --- a/service/submitqueue/demo/workload.go +++ b/service/submitqueue/demo/workload.go @@ -23,7 +23,8 @@ import ( // Five to ten shards produce both shared-directory and independent changes. const ( minShardDirs = 5 - maxShardDirs = 10 + // MaxShardDirs is also how many shard folders have TARGET files in the Tango demo sandbox. + MaxShardDirs = 10 ) func resolveFolders(tag string, configured int) int { @@ -31,7 +32,7 @@ func resolveFolders(tag string, configured int) int { return configured } sum := sha256.Sum256([]byte("folders#" + tag)) - return minShardDirs + int(sum[0])%(maxShardDirs-minShardDirs+1) + return minShardDirs + int(sum[0])%(MaxShardDirs-minShardDirs+1) } func changeShard(tag string, folders, change int) string { diff --git a/service/submitqueue/demo/workload_test.go b/service/submitqueue/demo/workload_test.go index d0f5f1b21..059f10d19 100644 --- a/service/submitqueue/demo/workload_test.go +++ b/service/submitqueue/demo/workload_test.go @@ -86,7 +86,7 @@ func TestResolveFolders(t *testing.T) { for minute := range 60 { folders := resolveFolders(fmt.Sprintf("0810-12%02d", minute), 0) assert.GreaterOrEqual(t, folders, minShardDirs) - assert.LessOrEqual(t, folders, maxShardDirs) + assert.LessOrEqual(t, folders, MaxShardDirs) seen[folders] = struct{}{} } assert.Greater(t, len(seen), 1, "runs must not all pick the same number") diff --git a/service/submitqueue/docker-compose.tango.yml b/service/submitqueue/docker-compose.tango.yml new file mode 100644 index 000000000..3fa57db01 --- /dev/null +++ b/service/submitqueue/docker-compose.tango.yml @@ -0,0 +1,28 @@ +# Compose overlay: answer conflict analysis with a fake Tango server. +# +# Layered over docker-compose.yml and docker-compose.git.yml, which it does not +# modify; the Makefile adds it when ANALYZER=tango: +# +# make local-submitqueue-start PROVIDER=git ANALYZER=tango +# +# The orchestrator reads profiles.tango.yaml instead of profiles.yaml, which +# selects the Tango analyzer for demo-queue. The fake serves Tango's gRPC +# protocol and derives changed targets from the TARGET files in the +# orchestrator's own repository copy, mounted read-only. + +services: + orchestrator-service: + environment: + - PROFILES_CONFIG_PATH=/etc/submitqueue/profiles.tango.yaml + + tangofake-service: + image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-tangofake-service:latest + build: + context: ${REPO_ROOT} + dockerfile: service/tangofake/server/Dockerfile + user: "${SQ_CONTAINER_USER:-0:0}" + command: ["/app/tangofake", "-repo", "/var/submitqueue/changerepos/demo", "-address", "0.0.0.0:8080"] + volumes: + # The same copy docker-compose.git.yml gives the orchestrator, so the fake + # sees every commit the orchestrator has fetched. + - ${SQ_ORCHESTRATOR_REPO_DIR:-orchestrator-changerepos}:/var/submitqueue/changerepos:ro diff --git a/service/submitqueue/orchestrator/server/BUILD.bazel b/service/submitqueue/orchestrator/server/BUILD.bazel index 6e186511c..20fa35e32 100644 --- a/service/submitqueue/orchestrator/server/BUILD.bazel +++ b/service/submitqueue/orchestrator/server/BUILD.bazel @@ -130,7 +130,10 @@ go_test( ], # The pinned git, for the tests that resolve a git change provider against a # real repository. - data = ["@git"], + data = [ + "//service/submitqueue/demo/provider/git:profiles.tango.yaml", + "@git", + ], embed = [":orchestrator_lib"], # keep env = { "SUBMITQUEUE_TEST_GIT": "$(location @git//:git)", diff --git a/service/submitqueue/orchestrator/server/config_test.go b/service/submitqueue/orchestrator/server/config_test.go index b84846df7..979234c7f 100644 --- a/service/submitqueue/orchestrator/server/config_test.go +++ b/service/submitqueue/orchestrator/server/config_test.go @@ -1030,3 +1030,9 @@ queues: assert.NotNil(t, analyzer) } } + +func TestDemoTangoProfileLoads(t *testing.T) { + cfg, err := loadProfilesConfig("../../demo/provider/git/profiles.tango.yaml") + require.NoError(t, err) + assert.True(t, cfg.usesTango()) +} diff --git a/tool/gitsandbox/main.go b/tool/gitsandbox/main.go index e758b15a8..6fd1362e9 100644 --- a/tool/gitsandbox/main.go +++ b/tool/gitsandbox/main.go @@ -51,15 +51,16 @@ func main() { checkoutDir := flag.String("checkout-dir", "", "directory Runway provisions its working trees in") repoName := flag.String("repo-name", "sandbox", "bare repository name, without the .git suffix") branch := flag.String("branch", "main", "target branch created by the seed commit") + targets := flag.Bool("targets", false, "commit the demo's TARGET files on the target branch if they are not there yet") flag.Parse() - if err := run(context.Background(), *git, *sandboxDir, *checkoutDir, *repoName, *branch); err != nil { + if err := run(context.Background(), *git, *sandboxDir, *checkoutDir, *repoName, *branch, *targets); err != nil { fmt.Fprintln(os.Stderr, "Error:", err) os.Exit(1) } } -func run(ctx context.Context, git, sandboxDir, checkoutDir, repoName, branch string) error { +func run(ctx context.Context, git, sandboxDir, checkoutDir, repoName, branch string, targets bool) error { if sandboxDir == "" { return fmt.Errorf("-sandbox-dir is required") } @@ -75,6 +76,12 @@ func run(ctx context.Context, git, sandboxDir, checkoutDir, repoName, branch str return err } + if targets { + if err := addTargets(ctx, resolved, bare, branch); err != nil { + return err + } + } + // Docker creates a missing bind-mount source itself, owned by root, which // on rootful Docker leaves a directory the host user cannot clean up. if checkoutDir != "" { @@ -164,9 +171,45 @@ func configure(ctx context.Context, git, bare string) error { return nil } -// seed commits an initial file on the target branch, through a throwaway clone -// because a bare repository has no working tree to commit from. +// seed commits an initial file on the target branch. func seed(ctx context.Context, git, bare, branch string) error { + return commitFiles(ctx, git, bare, branch, "seed the sandbox", map[string]string{seedFile: seedContents}) +} + +// targetsMarker is the file whose presence on the target branch means the +// targets have already been committed. +const targetsMarker = "demo/00/TARGET" + +// targetFiles is the target graph the Tango demo runs against: one target per +// shard folder the demo workload can create (demo.MaxShardDirs of them), as TARGET files in the form the +// fake Tango server reads. demo/00 and demo/01 are libraries that the other +// folders depend on, so a change to a library conflicts with a change to any +// folder built on it, though their paths never overlap. +func targetFiles() map[string]string { + files := make(map[string]string, 10) + for shard := 0; shard < 10; shard++ { + deps := "" + if shard >= 2 { + deps = fmt.Sprintf("//demo/%02d\n", shard%2) + } + files[fmt.Sprintf("demo/%02d/TARGET", shard)] = deps + } + return files +} + +// addTargets commits the demo's target graph on top of the target branch unless +// it is already there, so an existing sandbox keeps its history and gains the +// graph without being thrown away. +func addTargets(ctx context.Context, git, bare, branch string) error { + if err := gitexec.Run(ctx, git, bare, "cat-file", "-e", branch+":"+targetsMarker); err == nil { + return nil + } + return commitFiles(ctx, git, bare, branch, "add the demo target graph", targetFiles()) +} + +// commitFiles commits files on the target branch, through a throwaway clone +// because a bare repository has no working tree to commit from. +func commitFiles(ctx context.Context, git, bare, branch, message string, files map[string]string) error { work, err := os.MkdirTemp("", "sq-sandbox-seed-") if err != nil { return fmt.Errorf("could not create a working clone: %w", err) @@ -188,13 +231,19 @@ func seed(ctx context.Context, git, bare, branch string) error { } } - if err := os.WriteFile(filepath.Join(work, seedFile), []byte(seedContents), 0o644); err != nil { - return fmt.Errorf("could not write the seed file: %w", err) + for name, contents := range files { + full := filepath.Join(work, name) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return fmt.Errorf("could not create the directory for %s: %w", name, err) + } + if err := os.WriteFile(full, []byte(contents), 0o644); err != nil { + return fmt.Errorf("could not write %s: %w", name, err) + } } - if err := gitexec.Run(ctx, git, work, "add", seedFile); err != nil { + if err := gitexec.Run(ctx, git, work, "add", "-A"); err != nil { return err } - if err := gitexec.Run(ctx, git, work, "commit", "-m", "seed the sandbox"); err != nil { + if err := gitexec.Run(ctx, git, work, "commit", "-m", message); err != nil { return err } return gitexec.Run(ctx, git, work, "push", "origin", branch) diff --git a/tool/gitsandbox/main_test.go b/tool/gitsandbox/main_test.go index ca1245bbe..b8e5457bb 100644 --- a/tool/gitsandbox/main_test.go +++ b/tool/gitsandbox/main_test.go @@ -167,14 +167,14 @@ func TestProvision_AppliesSettingsToAnExistingSandbox(t *testing.T) { } func TestRun_RequiresASandboxDirectory(t *testing.T) { - err := run(context.Background(), gitexectest.Git(t), "", "", "sandbox", "main") + err := run(context.Background(), gitexectest.Git(t), "", "", "sandbox", "main", false) require.Error(t, err) } func TestRun_CreatesTheCheckoutDirectory(t *testing.T) { checkout := filepath.Join(t.TempDir(), "checkouts") - err := run(context.Background(), gitexectest.Git(t), t.TempDir(), checkout, "sandbox", "main") + err := run(context.Background(), gitexectest.Git(t), t.TempDir(), checkout, "sandbox", "main", false) require.NoError(t, err) info, err := os.Stat(checkout) @@ -215,3 +215,37 @@ func TestProvision_SucceedsAfterAFailedAttempt(t *testing.T) { _, err = gitexec.Output(ctx, git, bare, "rev-parse", "refs/heads/main") assert.NoError(t, err) } + +func TestAddTargets(t *testing.T) { + git := gitexectest.Git(t) + ctx := context.Background() + bare := filepath.Join(t.TempDir(), "sandbox.git") + _, err := provision(ctx, git, bare, "main") + require.NoError(t, err) + seeded, err := gitexec.Output(ctx, git, bare, "rev-parse", "refs/heads/main") + require.NoError(t, err) + + require.NoError(t, addTargets(ctx, git, bare, "main")) + added, err := gitexec.Output(ctx, git, bare, "rev-parse", "refs/heads/main") + require.NoError(t, err) + + parent, err := gitexec.Output(ctx, git, bare, "rev-parse", "refs/heads/main^") + require.NoError(t, err) + assert.Equal(t, seeded, parent, "the graph is committed on top of the existing history") + + for file, want := range map[string]string{ + "demo/00/TARGET": "", + "demo/01/TARGET": "", + "demo/04/TARGET": "//demo/00\n", + "demo/09/TARGET": "//demo/01\n", + } { + got, err := gitexec.Output(ctx, git, bare, "show", "refs/heads/main:"+file) + require.NoError(t, err, file) + assert.Equal(t, strings.TrimSpace(want), got, file) + } + + require.NoError(t, addTargets(ctx, git, bare, "main")) + again, err := gitexec.Output(ctx, git, bare, "rev-parse", "refs/heads/main") + require.NoError(t, err) + assert.Equal(t, added, again, "a sandbox that already has the graph is left as it is") +}