Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 158 additions & 0 deletions lab/internal/onboarding/bootstrap_planner_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package onboarding

import (
"context"
"os"
"path/filepath"
"strings"
"testing"

"github.com/DanielNetzer/forgecell/lab/internal/harness"
"github.com/DanielNetzer/forgecell/lab/internal/resources"
)

// prepareExplicitPlanner runs a fresh-Lab init --harness coding with an explicit
// Planner over the given candidates.
func prepareExplicitPlanner(t *testing.T, coding, planner string, candidates ...Candidate) (Outcome, string, error) {
t.Helper()
root, run := repoFixture(t)
dir := filepath.Join(t.TempDir(), "lab")
out, err := Prepare(context.Background(), InitOptions{Cwd: root, LabDir: dir, Launcher: "/installed/forgecell", Preferred: coding, Planner: planner, Candidates: candidates, Runner: run})
return out, dir, err
}

// assertNothingSaved checks a refused Prepare returned no proposal and wrote no
// assays and no resource configuration.
func assertNothingSaved(t *testing.T, out Outcome, dir string) {
t.Helper()
if out.Proposal != nil {
t.Fatalf("refusal returned a proposal: %+v", out.Proposal)
}
if entries, e := os.ReadDir(filepath.Join(dir, "assays")); e == nil && len(entries) > 0 || e != nil && !os.IsNotExist(e) {
t.Fatalf("refusal wrote assays: %v %v", entries, e)
}
if _, e := os.Stat(filepath.Join(dir, resources.ConfigFile)); !os.IsNotExist(e) {
t.Fatalf("refusal wrote %s: %v", resources.ConfigFile, e)
}
}

func allReady() []Candidate {
return []Candidate{bootstrapCandidate("cursor", "/bin/cursor-agent", "ready"), bootstrapCandidate("claude-code", "/bin/claude", "ready"), bootstrapCandidate("codex", "/bin/codex", "ready")}
}

func TestBootstrapExplicitPlannerCodex(t *testing.T) {
out, dir, err := prepareExplicitPlanner(t, "cursor", "codex", allReady()...)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(out.Note, "the planner you chose") {
t.Fatalf("note does not say the planner was chosen: %s", out.Note)
}
assertPlannedBy(t, out, dir, "codex", "/bin/codex")
}

func TestBootstrapExplicitPlannerClaudeCode(t *testing.T) {
candidates := []Candidate{bootstrapCandidate("cursor", "/bin/cursor-agent", "ready"), bootstrapCandidate("codex", "/bin/codex", "ready"), bootstrapCandidate("claude-code", "/bin/claude", "ready")}
out, dir, err := prepareExplicitPlanner(t, "cursor", "claude-code", candidates...)
if err != nil {
t.Fatal(err)
}
assertPlannedBy(t, out, dir, "claude-code", "/bin/claude")
}

func TestBootstrapExplicitPlannerRefusesUnreadyOrUnableToPlan(t *testing.T) {
cursorReason := harness.AdapterCapabilities("cursor").Analysis.Reason
for name, tc := range map[string]struct {
planner string
candidates []Candidate
want []string
}{
"cursor plans": {"cursor", allReady(), []string{"cannot be used for read-only ticket analysis", cursorReason, "no proposal saved"}},
"codex signed out": {"codex", []Candidate{bootstrapCandidate("cursor", "/bin/cursor-agent", "ready"), bootstrapCandidate("claude-code", "/bin/claude", "ready"), bootstrapCandidate("codex", "/bin/codex", "login-required")}, []string{"cannot bind harnesses.analysis to codex", "codex is login-required.", "no proposal saved"}},
"claude no binary": {"claude-code", []Candidate{bootstrapCandidate("cursor", "/bin/cursor-agent", "ready"), bootstrapCandidate("claude-code", "", "ready")}, []string{"cannot bind harnesses.analysis to claude-code", "no proposal saved"}},
"not discovered": {"codex", []Candidate{bootstrapCandidate("cursor", "/bin/cursor-agent", "ready"), bootstrapCandidate("claude-code", "/bin/claude", "ready")}, []string{"no such harness was discovered", "no proposal saved"}},
"unknown harness": {"other", allReady(), []string{"no such harness was discovered", "no proposal saved"}},
"coding plans self": {"codex", allReady(), nil},
} {
t.Run(name, func(t *testing.T) {
coding := "cursor"
if name == "coding plans self" {
coding = "claude-code"
tc.want = []string{"claude-code can do read-only ticket analysis itself", "no proposal saved"}
}
out, dir, err := prepareExplicitPlanner(t, coding, tc.planner, tc.candidates...)
if err == nil {
t.Fatalf("planner %s was accepted: %+v", tc.planner, out)
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("refusal lacks %q: %v", want, err)
}
}
assertNothingSaved(t, out, dir)
})
}
}

func TestBootstrapExplicitPlannerOnlyForABootstrapWithHarness(t *testing.T) {
for name, o := range map[string]InitOptions{
"no harness": {Planner: "codex"},
"enable review": {Preferred: "cursor", Planner: "codex", EnableReview: true},
"role": {Preferred: "cursor", Planner: "codex", RoleHarness: true, Role: "analysis"},
"rebind": {Rebind: "cursor", Planner: "codex"},
} {
t.Run(name, func(t *testing.T) {
root, run := repoFixture(t)
dir := filepath.Join(t.TempDir(), "lab")
o.Cwd, o.LabDir, o.Launcher, o.Candidates, o.Runner = root, dir, "/installed/forgecell", allReady(), run
out, err := Prepare(context.Background(), o)
if err == nil || !strings.Contains(err.Error(), "a planner is chosen only for a GitHub bootstrap") {
t.Fatalf("planner with %s: %v", name, err)
}
assertNothingSaved(t, out, dir)
})
}
// A Lab that already has a Formula takes no bootstrap planner.
root, run := repoFixture(t)
dir := filepath.Join(t.TempDir(), "lab")
out, err := Prepare(context.Background(), InitOptions{Cwd: root, LabDir: dir, Launcher: "/installed/forgecell", Preferred: "cursor", Planner: "codex", Candidates: allReady(), Runner: run})
if err != nil {
t.Fatal(err)
}
if _, err = Review(dir, out.Proposal.ID, "approve"); err != nil {
t.Fatal(err)
}
if _, err = Prepare(context.Background(), InitOptions{Cwd: root, LabDir: dir, Launcher: "/installed/forgecell", Preferred: "cursor", Planner: "claude-code", Candidates: allReady(), Runner: run}); err == nil || !strings.Contains(err.Error(), "only for a bootstrap") {
t.Fatalf("planner on an approved Lab: %v", err)
}
}

func TestBootstrapExplicitPlannerDefaultMatchesTheEngine(t *testing.T) {
for name, tc := range map[string]struct {
candidates []Candidate
want string
}{
"claude first": {allReady(), "claude-code"},
"codex second": {[]Candidate{bootstrapCandidate("claude-code", "/bin/claude", "login-required"), bootstrapCandidate("codex", "/bin/codex", "ready")}, "codex"},
"none ready": {[]Candidate{bootstrapCandidate("claude-code", "", "missing"), bootstrapCandidate("cursor", "/bin/cursor-agent", "ready")}, ""},
"none at all": {nil, ""},
"relative paths": {[]Candidate{bootstrapCandidate("claude-code", "bin/claude", "ready")}, ""},
} {
t.Run(name, func(t *testing.T) {
if got := DefaultBootstrapPlanner(tc.candidates, "/installed/forgecell"); got != tc.want {
t.Fatalf("default planner %q, want %q", got, tc.want)
}
})
}
if DefaultBootstrapPlanner(allReady(), "") != "" {
t.Fatal("a default planner without a launcher")
}
for id, want := range map[string]string{"claude-code": "supported", "codex": "supported", "cursor": "unsupported"} {
if got := PlanningCapability(id); got.State != want || got.Reason == "" {
t.Fatalf("%s planning capability %+v, want %s", id, got, want)
}
}
if got := PlanningCapability("cursor").Reason; got != harness.AdapterCapabilities("cursor").Analysis.Reason {
t.Fatalf("cursor reason %q", got)
}
}
74 changes: 70 additions & 4 deletions lab/internal/onboarding/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ type InitOptions struct {
// RoleHarness marks an explicit --harness ID --role ROLE: Preferred is proposed
// as the harnesses.<Role> override only, declaring the role when it is absent.
RoleHarness bool
// Planner chooses the harnesses.analysis binding of a bootstrap whose coding
// harness cannot do read-only ticket analysis; empty keeps the first ready
// planner (claude-code, then codex).
Planner string
// RefreshBaseline proposes a baseline-only change: requiredChecks re-derived
// from the current pull-request workflows of the approved Formula's repository.
RefreshBaseline bool
Expand Down Expand Up @@ -447,20 +451,68 @@ func bootstrapPlanner(candidates []Candidate, coding string, command []string, l
if analysis.State != "unsupported" {
return nil, nil, nil
}
if planner, plannerCommand := firstBootstrapPlanner(candidates, launcher, lab); planner != nil {
return planner, plannerCommand, nil
}
return nil, nil, fmt.Errorf("cannot propose a bootstrap Formula that codes with %s: it cannot be used for read-only ticket analysis: %s; planning needs a ready Claude Code or Codex harness and neither is ready; install or sign in to Claude Code or Codex, then rerun forgecell init; no proposal saved", coding, analysis.Reason)
}

// firstBootstrapPlanner is the first ready, discovered bootstrap planner whose
// adapter can do read-only ticket analysis, with its native command, or nil.
func firstBootstrapPlanner(candidates []Candidate, launcher, lab string) (*Candidate, []string) {
for _, id := range bootstrapPlanners {
for i := range candidates {
if candidates[i].ID != id || candidates[i].Readiness != "ready" || candidates[i].Executable == "" {
continue
}
if planner, err := roleHarnessTarget(candidates, id, launcher, formula.AnalysisHarness, lab); err == nil {
return &candidates[i], planner, nil
return &candidates[i], planner
}
}
}
return nil, nil, fmt.Errorf("cannot propose a bootstrap Formula that codes with %s: it cannot be used for read-only ticket analysis: %s; planning needs a ready Claude Code or Codex harness and neither is ready; install or sign in to Claude Code or Codex, then rerun forgecell init; no proposal saved", coding, analysis.Reason)
return nil, nil
}

// DefaultBootstrapPlanner is the harness a bootstrap that codes with a harness
// unable to plan binds harnesses.analysis to when no Planner is chosen, or ""
// when none is ready. It executes nothing.
func DefaultBootstrapPlanner(candidates []Candidate, launcher string) string {
if planner, _ := firstBootstrapPlanner(candidates, launcher, ""); planner != nil {
return planner.ID
}
return ""
}

// PlanningCapability is the static support of the native adapter id for
// read-only ticket analysis. It executes nothing.
func PlanningCapability(id string) harness.Capability {
return roleCapability(formula.AnalysisHarness, formula.Binding{Binding: id, Command: []string{"", "__adapter", id, ""}})
}

// explicitBootstrapPlanner is the chosen Planner of a bootstrap whose coding
// binding cannot do read-only ticket analysis. It refuses a coding binding that
// can plan itself, and a planner that is not ready or cannot plan, quoting the
// adapter's reason. It executes nothing.
func explicitBootstrapPlanner(candidates []Candidate, coding string, command []string, id, launcher, lab string) (*Candidate, []string, error) {
if analysis := roleCapability(formula.AnalysisHarness, formula.Binding{Binding: coding, Command: command}); analysis.State != "unsupported" {
return nil, nil, fmt.Errorf("%s can do read-only ticket analysis itself, so a bootstrap that codes with it takes no separate planner; no proposal saved", coding)
}
plannerCommand, err := roleHarnessTarget(candidates, id, launcher, formula.AnalysisHarness, lab)
if err != nil {
return nil, nil, err
}
for i := range candidates {
if candidates[i].ID == id {
return &candidates[i], plannerCommand, nil
}
}
return nil, nil, fmt.Errorf("cannot bind harnesses.analysis to %q: no such harness was discovered; no proposal saved", id)
}

func Prepare(ctx context.Context, o InitOptions) (out Outcome, err error) {
if o.Planner != "" && (o.pluginInitRequested() || o.Preferred == "" || o.Rebind != "" || o.RoleHarness || o.EnableReview || o.RefreshBaseline || o.ModelSet || o.Role != "" || o.MetaHarness != "" || o.MetaCommand != nil) {
return out, fmt.Errorf("a planner is chosen only for a GitHub bootstrap with --harness ID; do not combine it with plugins, --rebind, --role, --enable-review, --refresh-baseline, --model, --meta-harness or --meta-command; no proposal saved")
}
if o.pluginInitRequested() {
return preparePluginInit(ctx, o)
}
Expand Down Expand Up @@ -546,6 +598,9 @@ func Prepare(ctx context.Context, o InitOptions) (out Outcome, err error) {
if o.RoleHarness {
return out, fmt.Errorf("a proposal is already pending; review, approve or dismiss it with init --lab %s --review/--approve/--dismiss %s before --harness --role", formula.ShellWord(o.LabDir), pending.ID)
}
if o.Planner != "" {
return out, fmt.Errorf("a proposal is already pending; review, approve or dismiss it with init --lab %s --review/--approve/--dismiss %s before choosing a planner", formula.ShellWord(o.LabDir), pending.ID)
}
f, e := formula.Parse([]byte(pending.YAML))
if e != nil {
return out, e
Expand Down Expand Up @@ -587,6 +642,9 @@ func Prepare(ctx context.Context, o InitOptions) (out Outcome, err error) {
if o.RoleHarness && active.Formula.ID == "" {
return out, fmt.Errorf("--harness --role needs an approved Formula; run init --lab %s and approve the bootstrap proposal first", formula.ShellWord(o.LabDir))
}
if o.Planner != "" && active.Formula.ID != "" {
return out, fmt.Errorf("a planner is chosen only for a bootstrap; this Lab already has a Formula, so run forgecell init --harness ID --role analysis after approval; no proposal saved")
}
if active.Formula.ID != "" {
approved, e := formula.Load(o.LabDir, "")
if e != nil || !approved.Approved {
Expand Down Expand Up @@ -691,7 +749,11 @@ func Prepare(ctx context.Context, o InitOptions) (out Outcome, err error) {
// with a planning harness, or refuses before anything is saved.
var planner *Candidate
var plannerCommand []string
if !active.Approved {
if !active.Approved && o.Planner != "" {
if planner, plannerCommand, err = explicitBootstrapPlanner(out.Candidates, selected.ID, command, o.Planner, o.Launcher, o.LabDir); err != nil {
return out, err
}
} else if !active.Approved {
if planner, plannerCommand, err = bootstrapPlanner(out.Candidates, selected.ID, command, o.Launcher, o.LabDir); err != nil {
return out, err
}
Expand Down Expand Up @@ -755,7 +817,11 @@ func Prepare(ctx context.Context, o InitOptions) (out Outcome, err error) {
out.Note = "Bootstrap Assay is read-only. Proposed checks are evidence, not executed commands. Approving binds the coding harness; runs still stop at human review."
if planner != nil {
recipe["harnesses"] = map[string]any{string(formula.AnalysisHarness): map[string]any{"binding": planner.ID, "command": plannerCommand}}
out.Note += fmt.Sprintf(" %s cannot do read-only ticket analysis, so harnesses.analysis binds planning to %s, the first ready harness that can (claude-code, then codex); %s still codes. To plan with another harness after approval, run forgecell init --harness ID --role analysis.", selected.ID, planner.ID, selected.ID)
chosen := "the first ready harness that can (claude-code, then codex)"
if o.Planner != "" {
chosen = "the planner you chose"
}
out.Note += fmt.Sprintf(" %s cannot do read-only ticket analysis, so harnesses.analysis binds planning to %s, %s; %s still codes. To plan with another harness after approval, run forgecell init --harness ID --role analysis.", selected.ID, planner.ID, chosen, selected.ID)
}
if len(baseline) > 0 {
recipe["requiredChecks"] = baseline
Expand Down
Loading
Loading