From 44f475812deffa70380aa37edd17d17a502934f1 Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 13 Oct 2026 16:19:31 +0000
Subject: [PATCH] refactor: share workspace provenance, readiness, delete, and
 429 backoff steps

Refs #153
---
 .github/workflows/ci.yaml                     |   1 +
 internal/controller/coder_access.go           |  37 +++
 internal/controller/coder_workspace_steps.go  | 304 ++++++++++++++++++
 .../controller/codertemplatetest_confirm.go   |  54 +---
 .../codertemplatetest_controller.go           |  45 +--
 .../controller/codertemplatetest_create.go    |  22 +-
 .../controller/codertemplatetest_delete.go    | 118 +------
 .../controller/codertemplatetest_readiness.go |  55 +---
 8 files changed, 361 insertions(+), 275 deletions(-)
 create mode 100644 internal/controller/coder_workspace_steps.go

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 32e934bef..db3c9f2cd 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -88,6 +88,7 @@ jobs:
               # Shared Coder access that CoderTemplateTest and later pool controllers call.
               - 'internal/controller/controlplane_access.go'
               - 'internal/controller/coder_access.go'
+              - 'internal/controller/coder_workspace_steps.go'
               - 'internal/app/controllerapp/**'
               - 'config/e2e/**'
               - 'deploy/**'
diff --git a/internal/controller/coder_access.go b/internal/controller/coder_access.go
index 858d93470..84e507987 100644
--- a/internal/controller/coder_access.go
+++ b/internal/controller/coder_access.go
@@ -23,6 +23,43 @@ import (
 // and version resolution. CoderTemplateTest uses them today, and
 // CoderWorkspacePool (#153) will too.
 
+// coderAnswerFor is the rule for every Coder answer a controller trusts:
+// the answer names the ID or name that the controller asked for. Anything else
+// means Coder, a proxy, or the SDK is broken, so it is an assertion failure.
+func coderAnswerFor(kind, asked, answered string) error {
+	if asked == answered {
+		return nil
+	}
+	return &coderAnswerError{msg: fmt.Sprintf("assertion failed: Coder answered %s %q when asked for %q", kind, answered, asked)}
+}
+
+type coderAnswerError struct{ msg string }
+
+func (e *coderAnswerError) Error() string { return e.msg }
+
+func isCoderNotFound(err error) bool {
+	return coderStatus(err) == http.StatusNotFound
+}
+
+// coderErrorSummary holds the status and Coder's short message only, never
+// details, validation errors, or job logs.
+func coderErrorSummary(err error) string {
+	var sdkErr *codersdk.Error
+	if errors.As(err, &sdkErr) {
+		return fmt.Sprintf("Coder answered %d: %s", sdkErr.StatusCode(), sdkErr.Message)
+	}
+	return err.Error()
+}
+
+// coderStatus returns the HTTP status of a Coder answer, or 0 without one.
+func coderStatus(err error) int {
+	var sdkErr *codersdk.Error
+	if errors.As(err, &sdkErr) {
+		return sdkErr.StatusCode()
+	}
+	return 0
+}
+
 // coderStep is the outcome of a shared lookup that cannot go on: a wait,
 // when the input can still appear, or a failure, when it never will. Each
 // controller turns it into its own status.
diff --git a/internal/controller/coder_workspace_steps.go b/internal/controller/coder_workspace_steps.go
new file mode 100644
index 000000000..1d7d24a8c
--- /dev/null
+++ b/internal/controller/coder_workspace_steps.go
@@ -0,0 +1,304 @@
+package controller
+
+import (
+	"context"
+	"errors"
+	"math/rand/v2"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/google/uuid"
+)
+
+// This file holds the workspace steps that every controller which builds
+// workspaces shares: provenance after an uncertain create, the readiness rules
+// for one build, the delete step, and the HTTP 429 backoff. CoderTemplateTest
+// uses them today, and CoderWorkspacePool (#153) will too.
+
+const (
+	// coderBuildPoll is how often a controller reads a workspace while a
+	// build or a delete runs.
+	coderBuildPoll = 5 * time.Second
+
+	// The HTTP 429 backoff doubles from 2 s to its 2 m cap, with 20 % jitter,
+	// like the CoderProvisioner controller.
+	coderBackoffBase = 2 * time.Second
+	coderBackoffCap  = 2 * time.Minute
+
+	// The delete retry backoff doubles from 1 m to its 30 m cap. Both values
+	// are guesses, not measurements (#152 plan A3).
+	coderDeleteBackoffBase = time.Minute
+	coderDeleteBackoffCap  = 30 * time.Minute
+)
+
+// coderBackoff counts consecutive HTTP 429 answers per key, in memory, so a
+// restart starts over. The zero value is ready to use.
+type coderBackoff struct {
+	attempts sync.Map // Key -> consecutive HTTP 429 answers.
+}
+
+// next counts one more 429 for key and returns the wait before the next try.
+func (b *coderBackoff) next(key any) time.Duration {
+	attempts := 0
+	if v, ok := b.attempts.Load(key); ok {
+		attempts = v.(int)
+	}
+	b.attempts.Store(key, attempts+1)
+	backoff := coderBackoffCap
+	if attempts < 6 { // 2 s * 2^6 passes the cap.
+		backoff = min(coderBackoffBase<<attempts, coderBackoffCap)
+	}
+	jittered := time.Duration(float64(backoff) * (0.8 + 0.4*rand.Float64())) //nolint:gosec // Jitter needs no secure randomness.
+	return min(jittered, coderBackoffCap)
+}
+
+// forget drops the count for key.
+func (b *coderBackoff) forget(key any) {
+	b.attempts.Delete(key)
+}
+
+// workspaceProvenance returns the start build that proves the operator
+// created the workspace: organization and template match the pinned values,
+// and the operator user started the workspace's first build with the pinned
+// version. It returns nil without proof. It reads the newest 100 builds: a
+// workspace with more builds stays unproven, which keeps its finalizer.
+//
+// Coder v2.37.2 can answer the create request by claiming a prebuilt
+// workspace when the parameters match a preset (coderd/workspaces.go:670-685).
+// Build 1 is then the prebuild: a start build by the prebuilds system user.
+// Only Coder's prebuilds reconciler starts builds as that user
+// (enterprise/coderd/prebuilds/reconcile.go:1119-1122), and nobody can log in
+// or get an API key as it (coderd/apikey.go:62-66). The claim build is the
+// first build by anyone else: a start build by the operator with the
+// requested version (coderd/workspaces.go:769-785). Any other build between
+// the prebuild and the claim leaves the workspace unproven (fail-safe).
+func workspaceProvenance(
+	ctx context.Context, sdk *codersdk.Client, workspace codersdk.Workspace, operatorID uuid.UUID, organizationID, templateID, templateVersionID string,
+) (*codersdk.WorkspaceBuild, error) {
+	if workspace.OrganizationID.String() != organizationID || workspace.TemplateID.String() != templateID {
+		return nil, nil
+	}
+	builds, err := sdk.WorkspaceBuilds(ctx, codersdk.WorkspaceBuildsRequest{WorkspaceID: workspace.ID, Pagination: codersdk.Pagination{Limit: 100}})
+	if err != nil {
+		return nil, err
+	}
+	prebuildsUser := uuid.MustParse(codersdk.PrebuildsSystemUserID)
+	var first, claim *codersdk.WorkspaceBuild
+	for i := range builds {
+		b := &builds[i]
+		if err := coderAnswerFor("build workspace", workspace.ID.String(), b.WorkspaceID.String()); err != nil {
+			return nil, err
+		}
+		if b.BuildNumber == 1 {
+			first = b
+		}
+		if b.InitiatorID != prebuildsUser && (claim == nil || b.BuildNumber < claim.BuildNumber) {
+			claim = b
+		}
+	}
+	// Without build 1 the answer proves nothing. Without a prebuild, claim is
+	// build 1: a later start build by the operator does not prove who created
+	// the workspace.
+	if first == nil || claim == nil || claim.Transition != codersdk.WorkspaceTransitionStart || claim.InitiatorID != operatorID ||
+		claim.TemplateVersionID.String() != templateVersionID {
+		return nil, nil
+	}
+	if claim.ID == uuid.Nil {
+		return nil, &coderAnswerError{msg: "assertion failed: Coder answered a start build without an ID"}
+	}
+	return claim, nil
+}
+
+// workspaceBuildReadiness applies the readiness rules (#152 plan 2.5) to the
+// start build of workspace name, after the caller checked that it is the
+// expected build. Nil means the build succeeded and every top-level agent is
+// connected and ready. Workspace.Health.Healthy is never the success signal:
+// it is true while agents still run their startup scripts.
+func workspaceBuildReadiness(build codersdk.WorkspaceBuild, name string) *coderStep {
+	switch build.Job.Status {
+	case codersdk.ProvisionerJobSucceeded:
+	case codersdk.ProvisionerJobFailed:
+		// Only the error code: Job.Error can hold Terraform output and secrets.
+		if build.Job.ErrorCode == "" {
+			return coderFail("BuildFailed", "The start build of workspace %s failed.", name)
+		}
+		return coderFail("BuildFailed", "The start build of workspace %s failed (error code %s).", name, build.Job.ErrorCode)
+	case codersdk.ProvisionerJobCanceling, codersdk.ProvisionerJobCanceled:
+		// The controllers cancel only during cleanup.
+		return coderFail("BuildCanceled", "Someone else canceled the start build of workspace %s.", name)
+	default:
+		return coderWait("WaitingForBuild", "The start build of workspace %s is %s.", name, build.Job.Status)
+	}
+
+	var agents []codersdk.WorkspaceAgent
+	for _, resource := range build.Resources {
+		for _, agent := range resource.Agents {
+			if !agent.ParentID.Valid { // Devcontainer sub-agents have a parent.
+				agents = append(agents, agent)
+			}
+		}
+	}
+	if len(agents) == 0 {
+		// Coder reports a workspace without agents as healthy: a false pass.
+		return coderFail("NoAgents", "Workspace %s has no agents, so nothing proves that it works.", name)
+	}
+	// Messages name agents by ID: a template can derive agent names from
+	// parameter values, which never go into status.
+	var waiting *codersdk.WorkspaceAgent
+	for i := range agents {
+		agent := &agents[i]
+		switch agent.LifecycleState {
+		case codersdk.WorkspaceAgentLifecycleStartError:
+			return coderFail("AgentStartError", "The startup script of agent %s failed.", agent.ID)
+		case codersdk.WorkspaceAgentLifecycleStartTimeout:
+			return coderFail("AgentStartTimeout", "The startup script of agent %s timed out.", agent.ID)
+		case codersdk.WorkspaceAgentLifecycleShuttingDown, codersdk.WorkspaceAgentLifecycleShutdownTimeout,
+			codersdk.WorkspaceAgentLifecycleShutdownError, codersdk.WorkspaceAgentLifecycleOff:
+			return coderFail("AgentStopped", "Agent %s is %s.", agent.ID, agent.LifecycleState)
+		}
+		switch {
+		case agent.Status == codersdk.WorkspaceAgentTimeout:
+			return coderFail("AgentConnectionTimeout", "Agent %s did not connect in time.", agent.ID)
+		case waiting == nil && (agent.Status != codersdk.WorkspaceAgentConnected || agent.LifecycleState != codersdk.WorkspaceAgentLifecycleReady):
+			waiting = agent
+		}
+	}
+	// An agent that was ready and then disconnects makes the caller wait, not
+	// fail: disconnects can be short (#152 plan A10).
+	if waiting != nil {
+		return coderWait("WaitingForAgents", "Agent %s is %s and %s.", waiting.ID, waiting.Status, waiting.LifecycleState)
+	}
+	return nil
+}
+
+// workspaceDeleteState is what a controller stores to delete one workspace
+// safely across reconciles and restarts.
+type workspaceDeleteState struct {
+	workspaceID    uuid.UUID
+	workspaceName  string
+	deleteBuildID  string // The delete build this controller started or saw running.
+	deleteAttempts int32  // Failed delete builds counted so far, each once.
+}
+
+// workspaceDeleteResult is the outcome of one delete step.
+type workspaceDeleteResult struct {
+	deleted bool // A delete build succeeded: the workspace is gone.
+	// failedBuild is true when this step counted a failed delete build.
+	failedBuild bool
+	wait        *coderStep // Why the workspace still exists.
+	requeue     time.Duration
+}
+
+// deleteCoderWorkspace runs one delete step on state's workspace and updates
+// state. It acts on the latest build: it cancels an active start or stop
+// build, waits for an active delete build, and retries a failed delete build
+// after a backoff. It never cancels a delete build and never sends an orphan
+// delete. countEveryFailure counts a failed delete build that state does not
+// name, for example one whose ID a failed status write lost, or someone
+// else's.
+func deleteCoderWorkspace(
+	ctx context.Context, sdk *codersdk.Client, state *workspaceDeleteState, now time.Time, countEveryFailure bool,
+) (workspaceDeleteResult, error) {
+	if state == nil || state.workspaceID == uuid.Nil {
+		return workspaceDeleteResult{}, errors.New("assertion failed: the delete step needs a workspace ID")
+	}
+	name, workspaceID := state.workspaceName, state.workspaceID
+	wait := func(requeue time.Duration, reason, format string, args ...any) (workspaceDeleteResult, error) {
+		return workspaceDeleteResult{wait: coderWait(reason, format, args...), requeue: requeue}, nil
+	}
+	// include_deleted: a workspace whose delete build succeeded stays
+	// readable. A 404 is not proof of deletion (#152 plan A9).
+	workspace, err := sdk.DeletedWorkspace(ctx, workspaceID)
+	if err != nil {
+		return workspaceDeleteResult{wait: coderStepUnavailable("get workspace", err), requeue: coderBuildPoll}, nil
+	}
+	build := workspace.LatestBuild
+	if err := errors.Join(coderAnswerFor("workspace", workspaceID.String(), workspace.ID.String()),
+		coderAnswerFor("latest build workspace", workspaceID.String(), build.WorkspaceID.String())); err != nil {
+		return workspaceDeleteResult{}, err
+	}
+
+	status := build.Job.Status
+	switch {
+	case build.Transition == codersdk.WorkspaceTransitionDelete && status == codersdk.ProvisionerJobSucceeded:
+		return workspaceDeleteResult{deleted: true}, nil
+	case build.Transition == codersdk.WorkspaceTransitionDelete && status.Active():
+		// Any initiator: a delete build by someone else also deletes it.
+		state.deleteBuildID = build.ID.String()
+		return wait(coderBuildPoll, "Deleting", "The delete build of workspace %s is %s.", name, status)
+	case build.Transition == codersdk.WorkspaceTransitionDelete && status != codersdk.ProvisionerJobFailed && status != codersdk.ProvisionerJobCanceled:
+		// An unknown status proves no failure: wait.
+		return wait(coderBuildPoll, "Deleting", "The delete build of workspace %s is %s.", name, status)
+	case build.Transition == codersdk.WorkspaceTransitionDelete && (build.ID.String() == state.deleteBuildID || countEveryFailure):
+		// Count each failed delete build once. The caller stores the count
+		// before any new delete build.
+		state.deleteAttempts++
+		state.deleteBuildID = ""
+		retryAt := coderDeleteRetryAt(build, state.deleteAttempts, now)
+		// At least 1 s: the count must be stored before the next delete build.
+		result, _ := wait(max(retryAt.Sub(now), time.Second), "DeleteRetrying", "Delete build %d of workspace %s ended %s. The controller retries at %s.",
+			state.deleteAttempts, name, status, retryAt.UTC().Format(time.RFC3339))
+		result.failedBuild = true
+		return result, nil
+	case build.Transition == codersdk.WorkspaceTransitionDelete:
+		// Already counted, or someone else's: retry after the backoff.
+		if retryAt := coderDeleteRetryAt(build, state.deleteAttempts, now); now.Before(retryAt) {
+			return wait(retryAt.Sub(now), "DeleteRetrying", "The last delete build of workspace %s ended %s. The controller retries at %s.", name, status, retryAt.UTC().Format(time.RFC3339))
+		}
+	case status == codersdk.ProvisionerJobPending || status == codersdk.ProvisionerJobRunning:
+		// expect_status makes Coder refuse the cancel when the job moved on.
+		err := sdk.CancelWorkspaceBuild(ctx, build.ID, codersdk.CancelWorkspaceBuildParams{ExpectStatus: codersdk.CancelWorkspaceBuildStatus(status)})
+		if err != nil && coderStatus(err) != http.StatusPreconditionFailed && coderStatus(err) != http.StatusBadRequest {
+			return workspaceDeleteResult{wait: coderStepUnavailable("cancel build", err), requeue: coderBuildPoll}, nil
+		}
+		return wait(coderBuildPoll, "Deleting", "The controller canceled the %s build of workspace %s before deleting it.", build.Transition, name)
+	case status == codersdk.ProvisionerJobCanceling:
+		return wait(coderBuildPoll, "Deleting", "The %s build of workspace %s is canceling.", build.Transition, name)
+	}
+
+	deleteBuild, err := sdk.CreateWorkspaceBuild(ctx, workspaceID, codersdk.CreateWorkspaceBuildRequest{Transition: codersdk.WorkspaceTransitionDelete})
+	switch {
+	case coderStatus(err) == http.StatusConflict:
+		// A build became active: read again.
+		return wait(coderBuildPoll, "Deleting", "Another build of workspace %s started. Reading again.", name)
+	case err != nil:
+		return workspaceDeleteResult{wait: coderStepUnavailable("create delete build", err), requeue: coderBuildPoll}, nil
+	}
+	if err := errors.Join(coderAnswerFor("delete build workspace", workspaceID.String(), deleteBuild.WorkspaceID.String()),
+		coderAnswerFor("delete build transition", string(codersdk.WorkspaceTransitionDelete), string(deleteBuild.Transition))); err != nil {
+		return workspaceDeleteResult{}, err
+	}
+	if deleteBuild.ID == uuid.Nil {
+		return workspaceDeleteResult{}, &coderAnswerError{msg: "assertion failed: Coder answered a delete build without an ID"}
+	}
+	state.deleteBuildID = deleteBuild.ID.String()
+	return wait(coderBuildPoll, "Deleting", "The controller started a delete build of workspace %s.", name)
+}
+
+// coderDeleteRetryAt is when a controller may replace a failed delete build:
+// its completion plus the backoff. Without a completion time it uses the
+// build's own stable times, so repeated reads agree on the retry time.
+func coderDeleteRetryAt(build codersdk.WorkspaceBuild, attempts int32, now time.Time) time.Time {
+	anchor := now
+	switch {
+	case build.Job.CompletedAt != nil:
+		anchor = *build.Job.CompletedAt
+	case !build.UpdatedAt.IsZero():
+		anchor = build.UpdatedAt
+	case !build.CreatedAt.IsZero():
+		anchor = build.CreatedAt
+	}
+	return anchor.Add(coderDeleteBackoff(attempts))
+}
+
+// coderDeleteBackoff is the wait after the given number of failed delete
+// builds: 1 m, 2 m, 4 m, and so on, capped at 30 m.
+func coderDeleteBackoff(attempts int32) time.Duration {
+	backoff := coderDeleteBackoffBase
+	for i := int32(1); i < attempts && backoff < coderDeleteBackoffCap; i++ {
+		backoff *= 2
+	}
+	return min(backoff, coderDeleteBackoffCap)
+}
diff --git a/internal/controller/codertemplatetest_confirm.go b/internal/controller/codertemplatetest_confirm.go
index 1a3206e98..792b18ba5 100644
--- a/internal/controller/codertemplatetest_confirm.go
+++ b/internal/controller/codertemplatetest_confirm.go
@@ -66,7 +66,7 @@ func (r *CoderTemplateTestReconciler) confirmCreate(
 		case coderStatus(err) != http.StatusGone:
 			return coderUnavailable("get workspace", err), nil
 		}
-		startBuild, err = provenance(ctx, sdk, tt, workspace, operator.ID)
+		startBuild, err = workspaceProvenance(ctx, sdk, workspace, operator.ID, tt.Status.OrganizationID, tt.Status.TemplateID, tt.Status.TemplateVersionID)
 		var answerErr *coderAnswerError
 		if errors.As(err, &answerErr) {
 			return nil, err
@@ -106,55 +106,3 @@ func (r *CoderTemplateTestReconciler) confirmCreate(
 	return templateTestFailNotCreated("CreateOutcomeUnknown",
 		"No workspace %s from this test appeared within %s of the create request.", tt.Status.WorkspaceName, templateTestSettleWindow), nil
 }
-
-// provenance returns the start build that proves this test created the
-// workspace: organization and template match the pinned values, and the
-// operator user started the workspace's first build with the pinned version.
-// It returns nil without proof. It reads the newest 100 builds: a workspace
-// with more builds stays unproven, which keeps its finalizer.
-//
-// Coder v2.37.2 can answer the create request by claiming a prebuilt
-// workspace when the parameters match a preset (coderd/workspaces.go:670-685).
-// Build 1 is then the prebuild: a start build by the prebuilds system user.
-// Only Coder's prebuilds reconciler starts builds as that user
-// (enterprise/coderd/prebuilds/reconcile.go:1119-1122), and nobody can log in
-// or get an API key as it (coderd/apikey.go:62-66). The claim build is the
-// first build by anyone else: a start build by the operator with the
-// requested version (coderd/workspaces.go:769-785). Any other build between
-// the prebuild and the claim leaves the workspace unproven (fail-safe).
-func provenance(
-	ctx context.Context, sdk *codersdk.Client, tt *coderv1alpha1.CoderTemplateTest, workspace codersdk.Workspace, operatorID uuid.UUID,
-) (*codersdk.WorkspaceBuild, error) {
-	if workspace.OrganizationID.String() != tt.Status.OrganizationID || workspace.TemplateID.String() != tt.Status.TemplateID {
-		return nil, nil
-	}
-	builds, err := sdk.WorkspaceBuilds(ctx, codersdk.WorkspaceBuildsRequest{WorkspaceID: workspace.ID, Pagination: codersdk.Pagination{Limit: 100}})
-	if err != nil {
-		return nil, err
-	}
-	prebuildsUser := uuid.MustParse(codersdk.PrebuildsSystemUserID)
-	var first, claim *codersdk.WorkspaceBuild
-	for i := range builds {
-		b := &builds[i]
-		if err := coderAnswerFor("build workspace", workspace.ID.String(), b.WorkspaceID.String()); err != nil {
-			return nil, err
-		}
-		if b.BuildNumber == 1 {
-			first = b
-		}
-		if b.InitiatorID != prebuildsUser && (claim == nil || b.BuildNumber < claim.BuildNumber) {
-			claim = b
-		}
-	}
-	// Without build 1 the answer proves nothing. Without a prebuild, claim is
-	// build 1: a later start build by the operator does not prove who created
-	// the workspace.
-	if first == nil || claim == nil || claim.Transition != codersdk.WorkspaceTransitionStart || claim.InitiatorID != operatorID ||
-		claim.TemplateVersionID.String() != tt.Status.TemplateVersionID {
-		return nil, nil
-	}
-	if claim.ID == uuid.Nil {
-		return nil, &coderAnswerError{msg: "assertion failed: Coder answered a start build without an ID"}
-	}
-	return claim, nil
-}
diff --git a/internal/controller/codertemplatetest_controller.go b/internal/controller/codertemplatetest_controller.go
index 6f889fc2f..2edf4ee5c 100644
--- a/internal/controller/codertemplatetest_controller.go
+++ b/internal/controller/codertemplatetest_controller.go
@@ -5,9 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/http"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/coder/coder/v2/codersdk"
@@ -32,7 +30,7 @@ const (
 	templateTestWorkspacePrefix  = "ktt-"
 	templateTestUIDHexLength     = 28
 	templateTestPendingPoll      = 15 * time.Second
-	templateTestRunningPoll      = 5 * time.Second
+	templateTestRunningPoll      = coderBuildPoll
 	templateTestDefaultTimeout   = int32(900)
 	templateTestMaxMessageLength = 256
 )
@@ -49,7 +47,7 @@ type CoderTemplateTestReconciler struct {
 	Scheme *runtime.Scheme
 	Clock  clock.PassiveClock
 
-	rateLimited sync.Map // types.NamespacedName -> consecutive HTTP 429 answers.
+	rateLimited coderBackoff // Keyed by types.NamespacedName.
 }
 
 // templateTestControlPlaneRefIndex indexes tests by the control plane they use.
@@ -124,7 +122,7 @@ func (r *CoderTemplateTestReconciler) Reconcile(ctx context.Context, req ctrl.Re
 	keepBackoff := false
 	defer func() {
 		if !keepBackoff {
-			r.rateLimited.Delete(req.NamespacedName)
+			r.rateLimited.forget(req.NamespacedName)
 		}
 	}()
 	tt := &coderv1alpha1.CoderTemplateTest{}
@@ -391,20 +389,6 @@ func templateTestWorkspaceName(uid types.UID) (string, error) {
 	return templateTestWorkspacePrefix + hex[:templateTestUIDHexLength], nil
 }
 
-// coderAnswerFor is the rule for every Coder answer the reconciler trusts:
-// the answer names the ID or name that the controller asked for. Anything else
-// means Coder, a proxy, or the SDK is broken, so it is an assertion failure.
-func coderAnswerFor(kind, asked, answered string) error {
-	if asked == answered {
-		return nil
-	}
-	return &coderAnswerError{msg: fmt.Sprintf("assertion failed: Coder answered %s %q when asked for %q", kind, answered, asked)}
-}
-
-type coderAnswerError struct{ msg string }
-
-func (e *coderAnswerError) Error() string { return e.msg }
-
 func templateTestDeadlineExceeded(tt *coderv1alpha1.CoderTemplateTest, lastReason, lastMessage string) *templateTestStep {
 	return templateTestFail("DeadlineExceeded", "The test did not finish within %ds. Last wait: %s: %s",
 		templateTestTimeoutSeconds(tt), lastReason, lastMessage)
@@ -421,10 +405,6 @@ func isTemplateTestFinal(phase string) bool {
 	return phase == coderv1alpha1.CoderTemplateTestPhaseSucceeded || phase == coderv1alpha1.CoderTemplateTestPhaseFailed
 }
 
-func isCoderNotFound(err error) bool {
-	return coderStatus(err) == http.StatusNotFound
-}
-
 // coderUnavailable keeps the test waiting after a failed Coder call. The
 // message holds the status and Coder's short message only, never details.
 func coderUnavailable(action string, err error) *templateTestStep {
@@ -440,25 +420,6 @@ func templateTestStepFrom(step *coderStep) *templateTestStep {
 	return &templateTestStep{failed: step.failed, reason: step.reason, message: step.message, rateLimited: step.rateLimited}
 }
 
-// coderErrorSummary holds the status and Coder's short message only, never
-// details, validation errors, or job logs.
-func coderErrorSummary(err error) string {
-	var sdkErr *codersdk.Error
-	if errors.As(err, &sdkErr) {
-		return fmt.Sprintf("Coder answered %d: %s", sdkErr.StatusCode(), sdkErr.Message)
-	}
-	return err.Error()
-}
-
-// coderStatus returns the HTTP status of a Coder answer, or 0 without one.
-func coderStatus(err error) int {
-	var sdkErr *codersdk.Error
-	if errors.As(err, &sdkErr) {
-		return sdkErr.StatusCode()
-	}
-	return 0
-}
-
 func truncateTemplateTestMessage(message string) string {
 	if len(message) <= templateTestMaxMessageLength {
 		return message
diff --git a/internal/controller/codertemplatetest_create.go b/internal/controller/codertemplatetest_create.go
index b6511d7d5..06e2ec4d9 100644
--- a/internal/controller/codertemplatetest_create.go
+++ b/internal/controller/codertemplatetest_create.go
@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"math/rand/v2"
 	"net"
 	"net/http"
 	"time"
@@ -18,11 +17,6 @@ import (
 	coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1"
 )
 
-const (
-	templateTestBackoffBase = 2 * time.Second
-	templateTestBackoffCap  = 2 * time.Minute
-)
-
 // createWorkspace writes the create marker, then sends the only create
 // request of this attempt. A marker write conflict ends the reconcile before
 // any request, so a decision made on a stale object never reaches Coder.
@@ -115,8 +109,8 @@ func templateTestFailNotCreated(reason, format string, args ...any) *templateTes
 }
 
 // retryAfter returns the requeue delay for a step that waits. After HTTP 429
-// it backs off from 2 s to 2 m with 20 % jitter, like the CoderProvisioner
-// controller. The attempt count lives in memory, so a restart starts over.
+// it backs off from 2 s to 2 m with 20 % jitter (coderBackoff). The attempt
+// count lives in memory, so a restart starts over.
 // Reconcile forgets it after any outcome other than a wait after 429.
 func (r *CoderTemplateTestReconciler) retryAfter(key types.NamespacedName, tt *coderv1alpha1.CoderTemplateTest, step *templateTestStep) time.Duration {
 	// A 429 backoff wins over a step's own poll (r4173060957).
@@ -129,15 +123,5 @@ func (r *CoderTemplateTestReconciler) retryAfter(key types.NamespacedName, tt *c
 		}
 		return templateTestPendingPoll
 	}
-	attempts := 0
-	if v, ok := r.rateLimited.Load(key); ok {
-		attempts = v.(int)
-	}
-	r.rateLimited.Store(key, attempts+1)
-	backoff := templateTestBackoffCap
-	if attempts < 6 { // 2 s * 2^6 passes the cap.
-		backoff = min(templateTestBackoffBase<<attempts, templateTestBackoffCap)
-	}
-	jittered := time.Duration(float64(backoff) * (0.8 + 0.4*rand.Float64())) //nolint:gosec // Jitter needs no secure randomness.
-	return min(jittered, templateTestBackoffCap)
+	return r.rateLimited.next(key)
 }
diff --git a/internal/controller/codertemplatetest_delete.go b/internal/controller/codertemplatetest_delete.go
index 4a932b508..f28de7002 100644
--- a/internal/controller/codertemplatetest_delete.go
+++ b/internal/controller/codertemplatetest_delete.go
@@ -2,9 +2,7 @@ package controller
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"net/http"
 	"time"
 
 	"github.com/coder/coder/v2/codersdk"
@@ -14,13 +12,6 @@ import (
 	coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1"
 )
 
-const (
-	// The delete retry backoff doubles from 1 m to its 30 m cap. Both values
-	// are guesses, not measurements (plan A3).
-	templateTestDeleteBackoffBase = time.Minute
-	templateTestDeleteBackoffCap  = 30 * time.Minute
-)
-
 // deleteResult is the outcome of one delete step.
 type deleteResult struct {
 	deleted bool // A delete build succeeded: the workspace is gone.
@@ -39,84 +30,21 @@ type deleteResult struct {
 func (r *CoderTemplateTestReconciler) deleteStep(
 	ctx context.Context, sdk *codersdk.Client, tt *coderv1alpha1.CoderTemplateTest, now time.Time, afterPass bool,
 ) (deleteResult, error) {
-	name := tt.Status.WorkspaceName
-	wait := func(requeue time.Duration, reason, format string, args ...any) (deleteResult, error) {
-		return deleteResult{wait: templateTestWait(reason, format, args...), requeue: requeue}, nil
-	}
 	workspaceID, err := uuid.Parse(tt.Status.WorkspaceID)
 	if err != nil || workspaceID == uuid.Nil {
 		return deleteResult{}, fmt.Errorf("assertion failed: template test %s/%s has workspace ID %q", tt.Namespace, tt.Name, tt.Status.WorkspaceID)
 	}
-	// include_deleted: a workspace whose delete build succeeded stays
-	// readable. A 404 is not proof of deletion (plan A9).
-	workspace, err := sdk.DeletedWorkspace(ctx, workspaceID)
-	if err != nil {
-		return deleteResult{wait: coderUnavailable("get workspace", err), requeue: templateTestRunningPoll}, nil
-	}
-	build := workspace.LatestBuild
-	if err := errors.Join(coderAnswerFor("workspace", workspaceID.String(), workspace.ID.String()),
-		coderAnswerFor("latest build workspace", workspaceID.String(), build.WorkspaceID.String())); err != nil {
-		return deleteResult{}, err
-	}
-
-	status := build.Job.Status
-	switch {
-	case build.Transition == codersdk.WorkspaceTransitionDelete && status == codersdk.ProvisionerJobSucceeded:
-		return deleteResult{deleted: true}, nil
-	case build.Transition == codersdk.WorkspaceTransitionDelete && status.Active():
-		// Any initiator: a delete build by someone else also deletes it.
-		tt.Status.DeleteBuildID = build.ID.String()
-		return wait(templateTestRunningPoll, "Deleting", "The delete build of workspace %s is %s.", name, status)
-	case build.Transition == codersdk.WorkspaceTransitionDelete && status != codersdk.ProvisionerJobFailed && status != codersdk.ProvisionerJobCanceled:
-		// An unknown status proves no failure: wait.
-		return wait(templateTestRunningPoll, "Deleting", "The delete build of workspace %s is %s.", name, status)
-	case build.Transition == codersdk.WorkspaceTransitionDelete && (build.ID.String() == tt.Status.DeleteBuildID || afterPass):
-		// Count each failed delete build once. The status write stores the
-		// count before any new delete build. After a pass, every failed
-		// delete build counts and fails the test, even one whose ID a failed
-		// status write lost, or someone else's. The test has counted none
-		// yet, and the same status write makes it final, so it counts once.
-		tt.Status.DeleteAttempts++
-		tt.Status.DeleteBuildID = ""
-		retryAt := templateTestDeleteRetryAt(build, tt.Status.DeleteAttempts, now)
-		// At least 1 s: the count must be stored before the next delete build.
-		result, _ := wait(max(retryAt.Sub(now), time.Second), "DeleteRetrying", "Delete build %d of workspace %s ended %s. The controller retries at %s.",
-			tt.Status.DeleteAttempts, name, status, retryAt.UTC().Format(time.RFC3339))
-		result.failedBuild = true
-		return result, nil
-	case build.Transition == codersdk.WorkspaceTransitionDelete:
-		// Already counted, or someone else's: retry after the backoff.
-		if retryAt := templateTestDeleteRetryAt(build, tt.Status.DeleteAttempts, now); now.Before(retryAt) {
-			return wait(retryAt.Sub(now), "DeleteRetrying", "The last delete build of workspace %s ended %s. The controller retries at %s.", name, status, retryAt.UTC().Format(time.RFC3339))
-		}
-	case status == codersdk.ProvisionerJobPending || status == codersdk.ProvisionerJobRunning:
-		// expect_status makes Coder refuse the cancel when the job moved on.
-		err := sdk.CancelWorkspaceBuild(ctx, build.ID, codersdk.CancelWorkspaceBuildParams{ExpectStatus: codersdk.CancelWorkspaceBuildStatus(status)})
-		if err != nil && coderStatus(err) != http.StatusPreconditionFailed && coderStatus(err) != http.StatusBadRequest {
-			return deleteResult{wait: coderUnavailable("cancel build", err), requeue: templateTestRunningPoll}, nil
-		}
-		return wait(templateTestRunningPoll, "Deleting", "The controller canceled the %s build of workspace %s before deleting it.", build.Transition, name)
-	case status == codersdk.ProvisionerJobCanceling:
-		return wait(templateTestRunningPoll, "Deleting", "The %s build of workspace %s is canceling.", build.Transition, name)
-	}
-
-	deleteBuild, err := sdk.CreateWorkspaceBuild(ctx, workspaceID, codersdk.CreateWorkspaceBuildRequest{Transition: codersdk.WorkspaceTransitionDelete})
-	switch {
-	case coderStatus(err) == http.StatusConflict:
-		// A build became active: read again.
-		return wait(templateTestRunningPoll, "Deleting", "Another build of workspace %s started. Reading again.", name)
-	case err != nil:
-		return deleteResult{wait: coderUnavailable("create delete build", err), requeue: templateTestRunningPoll}, nil
-	}
-	if err := errors.Join(coderAnswerFor("delete build workspace", workspaceID.String(), deleteBuild.WorkspaceID.String()),
-		coderAnswerFor("delete build transition", string(codersdk.WorkspaceTransitionDelete), string(deleteBuild.Transition))); err != nil {
-		return deleteResult{}, err
-	}
-	if deleteBuild.ID == uuid.Nil {
-		return deleteResult{}, &coderAnswerError{msg: "assertion failed: Coder answered a delete build without an ID"}
+	state := workspaceDeleteState{
+		workspaceID: workspaceID, workspaceName: tt.Status.WorkspaceName,
+		deleteBuildID: tt.Status.DeleteBuildID, deleteAttempts: tt.Status.DeleteAttempts,
 	}
-	tt.Status.DeleteBuildID = deleteBuild.ID.String()
-	return wait(templateTestRunningPoll, "Deleting", "The controller started a delete build of workspace %s.", name)
+	// After a pass, every failed delete build counts and fails the test, even
+	// one whose ID a failed status write lost, or someone else's. The test has
+	// counted none yet, and the same status write makes it final, so it
+	// counts once.
+	result, err := deleteCoderWorkspace(ctx, sdk, &state, now, afterPass)
+	tt.Status.DeleteBuildID, tt.Status.DeleteAttempts = state.deleteBuildID, state.deleteAttempts
+	return deleteResult{deleted: result.deleted, failedBuild: result.failedBuild, wait: templateTestStepFrom(result.wait), requeue: result.requeue}, err
 }
 
 // deleteAfterPass runs the delete steps of a passed test. The test succeeds
@@ -143,29 +71,3 @@ func (r *CoderTemplateTestReconciler) deleteAfterPass(
 	step.requeue = result.requeue
 	return step, nil
 }
-
-// templateTestDeleteRetryAt is when the controller may replace a failed delete
-// build: its completion plus the backoff. Without a completion time it uses
-// the build's own stable times, so repeated reads agree on the retry time.
-func templateTestDeleteRetryAt(build codersdk.WorkspaceBuild, attempts int32, now time.Time) time.Time {
-	anchor := now
-	switch {
-	case build.Job.CompletedAt != nil:
-		anchor = *build.Job.CompletedAt
-	case !build.UpdatedAt.IsZero():
-		anchor = build.UpdatedAt
-	case !build.CreatedAt.IsZero():
-		anchor = build.CreatedAt
-	}
-	return anchor.Add(templateTestDeleteBackoff(attempts))
-}
-
-// templateTestDeleteBackoff is the wait after the given number of failed
-// delete builds: 1 m, 2 m, 4 m, and so on, capped at 30 m.
-func templateTestDeleteBackoff(attempts int32) time.Duration {
-	backoff := templateTestDeleteBackoffBase
-	for i := int32(1); i < attempts && backoff < templateTestDeleteBackoffCap; i++ {
-		backoff *= 2
-	}
-	return min(backoff, templateTestDeleteBackoffCap)
-}
diff --git a/internal/controller/codertemplatetest_readiness.go b/internal/controller/codertemplatetest_readiness.go
index 42969d243..7637962af 100644
--- a/internal/controller/codertemplatetest_readiness.go
+++ b/internal/controller/codertemplatetest_readiness.go
@@ -49,59 +49,8 @@ func (r *CoderTemplateTestReconciler) checkReadiness(
 		return templateTestFail("WorkspaceChangedExternally", "Someone else started build %d of workspace %s.", build.BuildNumber, name), nil
 	}
 
-	switch build.Job.Status {
-	case codersdk.ProvisionerJobSucceeded:
-	case codersdk.ProvisionerJobFailed:
-		// Only the error code: Job.Error can hold Terraform output and secrets.
-		if build.Job.ErrorCode == "" {
-			return templateTestFail("BuildFailed", "The start build of workspace %s failed.", name), nil
-		}
-		return templateTestFail("BuildFailed", "The start build of workspace %s failed (error code %s).", name, build.Job.ErrorCode), nil
-	case codersdk.ProvisionerJobCanceling, codersdk.ProvisionerJobCanceled:
-		// This controller cancels only during cleanup, which comes with plan PR 5.
-		return templateTestFail("BuildCanceled", "Someone else canceled the start build of workspace %s.", name), nil
-	default:
-		return templateTestWait("WaitingForBuild", "The start build of workspace %s is %s.", name, build.Job.Status), nil
-	}
-
-	var agents []codersdk.WorkspaceAgent
-	for _, resource := range build.Resources {
-		for _, agent := range resource.Agents {
-			if !agent.ParentID.Valid { // Devcontainer sub-agents have a parent.
-				agents = append(agents, agent)
-			}
-		}
-	}
-	if len(agents) == 0 {
-		// Coder reports a workspace without agents as healthy: a false pass.
-		return templateTestFail("NoAgents", "Workspace %s has no agents, so nothing proves that it works.", name), nil
-	}
-	// Messages name agents by ID: a template can derive agent names from
-	// parameter values, which never go into status.
-	var waiting *codersdk.WorkspaceAgent
-	for i := range agents {
-		agent := &agents[i]
-		switch agent.LifecycleState {
-		case codersdk.WorkspaceAgentLifecycleStartError:
-			return templateTestFail("AgentStartError", "The startup script of agent %s failed.", agent.ID), nil
-		case codersdk.WorkspaceAgentLifecycleStartTimeout:
-			return templateTestFail("AgentStartTimeout", "The startup script of agent %s timed out.", agent.ID), nil
-		case codersdk.WorkspaceAgentLifecycleShuttingDown, codersdk.WorkspaceAgentLifecycleShutdownTimeout,
-			codersdk.WorkspaceAgentLifecycleShutdownError, codersdk.WorkspaceAgentLifecycleOff:
-			return templateTestFail("AgentStopped", "Agent %s is %s.", agent.ID, agent.LifecycleState), nil
-		}
-		switch {
-		case agent.Status == codersdk.WorkspaceAgentTimeout:
-			return templateTestFail("AgentConnectionTimeout", "Agent %s did not connect in time.", agent.ID), nil
-		case waiting == nil && (agent.Status != codersdk.WorkspaceAgentConnected || agent.LifecycleState != codersdk.WorkspaceAgentLifecycleReady):
-			waiting = agent
-		}
-	}
-	// An agent that was ready and then disconnects makes the test wait, not
-	// fail: disconnects can be short, and the deadline bounds the wait. After
-	// the pass, the controller reads no agents again (plan A10).
-	if waiting != nil {
-		return templateTestWait("WaitingForAgents", "Agent %s is %s and %s.", waiting.ID, waiting.Status, waiting.LifecycleState), nil
+	if step := workspaceBuildReadiness(build, name); step != nil {
+		return templateTestStepFrom(step), nil
 	}
 	tt.Status.AgentsReadyTime = &metav1.Time{Time: now}
 	return templateTestWait("AgentsReady", "Every agent of workspace %s is ready. The controller deletes the workspace next.", name), nil