From 6a14d5bb52b896989deadaa941ab0e154cd21ca0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 07:24:15 +0000 Subject: [PATCH 1/5] fix: trust only Coder's own 404 when deleting AI Gateway keys Change-Id: I9e7b6f5ea04db940c6bd982d0a861a9a811ee0f6 Signed-off-by: Thomas Kosiewski --- .../controller/coderaigatewaykey_delete.go | 46 +++++++++++-------- .../coderaigatewaykey_delete_test.go | 31 +++++++++++-- .../coderaigatewaykey_mismatch_test.go | 35 ++++++++++++++ .../controller/coderaigatewaykey_recover.go | 14 ++---- .../coderaigatewaykey_stall_test.go | 38 +++++++++++++++ 5 files changed, 130 insertions(+), 34 deletions(-) diff --git a/internal/controller/coderaigatewaykey_delete.go b/internal/controller/coderaigatewaykey_delete.go index 03ab799d2..9cb82b895 100644 --- a/internal/controller/coderaigatewaykey_delete.go +++ b/internal/controller/coderaigatewaykey_delete.go @@ -2,7 +2,7 @@ package controller import ( "context" - "net/http" + "slices" "time" "github.com/coder/coder/v2/codersdk" @@ -58,10 +58,10 @@ func (r *CoderAIGatewayKeyReconciler) deleteStep(ctx context.Context, key *coder var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, aiGatewayKeyBoundedDeleteTimeout) defer cancel() - if _, failure, err := r.deleteKeys(ctx, sdk, key); err != nil { + if _, left, failure, err := r.deleteKeys(ctx, sdk, key); err != nil { return false, nil, err } else if failure != nil { - return r.releaseWithoutDelete(ctx, key, "the one pass while CoderControlPlane is being deleted failed: "+failure.message), nil, nil + return r.releaseWithoutDelete(ctx, key, "the one pass while CoderControlPlane is being deleted failed: "+failure.message, left...), nil, nil } return true, nil, nil } @@ -81,7 +81,7 @@ func (r *CoderAIGatewayKeyReconciler) deleteStep(ctx context.Context, key *coder // The pass repeats until a LIST shows no key of this object, so the // release follows a LIST that proves it. for range 2 { - deleted, failure, err := r.deleteKeys(ctx, sdk, key) + deleted, _, failure, err := r.deleteKeys(ctx, sdk, key) if err != nil || failure != nil { return false, failure, err } else if deleted == 0 { @@ -93,27 +93,32 @@ func (r *CoderAIGatewayKeyReconciler) deleteStep(ctx context.Context, key *coder } // deleteKeys lists the keys of this object, deletes each one, and returns -// how many it deleted, or the failure that stopped it. -func (r *CoderAIGatewayKeyReconciler) deleteKeys(ctx context.Context, sdk *codersdk.Client, key *coderv1alpha1.CoderAIGatewayKey) (int, *aiGatewayKeyStep, error) { +// how many it deleted. On a failed DELETE it also returns the failure and +// the IDs of the listed keys that it did not delete. +func (r *CoderAIGatewayKeyReconciler) deleteKeys(ctx context.Context, sdk *codersdk.Client, key *coderv1alpha1.CoderAIGatewayKey) (int, []string, *aiGatewayKeyStep, error) { prefix, err := aiGatewayKeyNamePrefix(key.UID) if err != nil { - return 0, nil, err + return 0, nil, nil, err } listed, err := sdk.ListAIGatewayKeys(ctx) if err != nil { - return 0, aiGatewayKeyDeleteFailure("list AI Gateway keys", err), nil + return 0, nil, aiGatewayKeyDeleteFailure("list AI Gateway keys", err), nil } - deleted := 0 - for _, k := range ownAIGatewayKeys(prefix, listed) { - if serial, _ := aiGatewayKeySerial(prefix, k.Name); serial >= key.Status.NextSerial { - continue // No create of this object used this serial (plan I3). + own := slices.DeleteFunc(ownAIGatewayKeys(prefix, listed), func(k codersdk.AIGatewayKey) bool { + serial, _ := aiGatewayKeySerial(prefix, k.Name) + return serial >= key.Status.NextSerial // No create of this object used this serial (plan I3). + }) + for i, k := range own { + // Only Coder's own 404 proves that the key is gone (as for providers, #325). + if err := sdk.DeleteAIGatewayKey(ctx, k.ID); err != nil && !isCoderResourceNotFound(err) { + var left []string + for _, rest := range own[i:] { + left = append(left, rest.ID.String()) + } + return i, left, aiGatewayKeyDeleteFailure("delete key "+k.Name, err), nil } - if err := sdk.DeleteAIGatewayKey(ctx, k.ID); (err != nil && coderStatus(err) != http.StatusNotFound) || coderRouteMissing(err) { - return deleted, aiGatewayKeyDeleteFailure("delete key "+k.Name, err), nil - } - deleted++ } - return deleted, nil, nil + return len(own), nil, nil, nil } // aiGatewayKeyDeleteFailure classifies a failed call of the deletion like a @@ -131,8 +136,9 @@ func aiGatewayKeyDeleteFailure(action string, err error) *aiGatewayKeyStep { } // releaseWithoutDelete logs the key IDs that can stay in Coder, because -// nothing can delete them any more, and returns true (ControlPlaneGone). -func (r *CoderAIGatewayKeyReconciler) releaseWithoutDelete(ctx context.Context, key *coderv1alpha1.CoderAIGatewayKey, why string) bool { +// nothing can delete them any more, and returns true (ControlPlaneGone). left +// holds the IDs that a failed pass listed but did not delete. +func (r *CoderAIGatewayKeyReconciler) releaseWithoutDelete(ctx context.Context, key *coderv1alpha1.CoderAIGatewayKey, why string, left ...string) bool { var ids []string if current := key.Status.CurrentKey; current != nil { ids = append(ids, current.ID) @@ -141,6 +147,6 @@ func (r *CoderAIGatewayKeyReconciler) releaseWithoutDelete(ctx context.Context, ids = append(ids, retiring.ID) } log.FromContext(ctx).Info("Releasing the finalizer without deleting the keys in Coder: the keys can stay in Coder", - "reason", "ControlPlaneGone", "why", why, "keyIDs", ids, "nextSerial", key.Status.NextSerial) + "reason", "ControlPlaneGone", "why", why, "keyIDs", ids, "listedKeyIDs", left, "nextSerial", key.Status.NextSerial) return true } diff --git a/internal/controller/coderaigatewaykey_delete_test.go b/internal/controller/coderaigatewaykey_delete_test.go index 738e80c8f..ab12d0d6c 100644 --- a/internal/controller/coderaigatewaykey_delete_test.go +++ b/internal/controller/coderaigatewaykey_delete_test.go @@ -9,9 +9,13 @@ import ( "github.com/coder/coder/v2/codersdk" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" + "github.com/coder/coder-k8s/internal/controller" ) // rotatedKey issues the first key and rotates once, so Coder holds the @@ -89,6 +93,15 @@ func TestCoderAIGatewayKeyDeleteRetries(t *testing.T) { heal: func(*fakeCoder) {}, reason: "CoderUnavailable", requeue: time.Minute, }, + // A 404 that is not Coder's own (an HTML page of a proxy) proves + // nothing, so the key stays and the deletion retries. + "proxy 404": { + fail: func(f *fakeCoder) { + f.failNext(routeDeleteAIGatewayKey, fakeFault{Status: http.StatusNotFound, HTML: "404"}) + }, + heal: func(*fakeCoder) {}, + reason: "CoderUnavailable", requeue: time.Minute, + }, "premium feature": { fail: func(f *fakeCoder) { f.setAIGatewayFeature(false) }, heal: func(f *fakeCoder) { f.setAIGatewayFeature(true) }, stalled: true, reason: "LicenseRequired", requeue: 10 * time.Minute, @@ -147,15 +160,27 @@ func TestCoderAIGatewayKeyDeleteWithControlPlaneGoing(t *testing.T) { require.NoError(t, k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: env.namespace}})) holdControlPlane(ctx, t, env.namespace) require.NoError(t, k8sClient.Delete(ctx, got)) + var orphan codersdk.CreateAIGatewayKeyResponse if fail { + // An orphan that status does not name: only the pass's LIST knows it. + require.NoError(t, k8sClient.Get(ctx, client.ObjectKeyFromObject(got), got)) + editStatus(ctx, t, got, func(s *coderv1alpha1.CoderAIGatewayKeyStatus) { s.NextSerial = 3 }) + var err error + orphan, err = env.fake.client(t, 5*time.Second).CreateAIGatewayKey(ctx, codersdk.CreateAIGatewayKeyRequest{Name: aiGatewayKeyPrefix(got) + "2"}) + require.NoError(t, err) env.fake.failNext(routeDeleteAIGatewayKey, fakeFault{Status: http.StatusBadGateway}) } - gone, _ := env.reconcile(ctx, t, got) - require.Nil(t, gone, "the finalizer went inside the settle window") + r := &controller.CoderAIGatewayKeyReconciler{Client: env.client, APIReader: env.client, Scheme: scheme, Clock: env.clock} + logCtx, logs := capturingContext(ctx) + _, err := r.Reconcile(logCtx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(got)}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(got), got)), "the finalizer went inside the settle window") require.Equal(t, 1, env.fake.requestCount(routeAIGatewayKeys)-2, "one LIST") if fail { require.Equal(t, 1, env.fake.requestCount(routeDeleteAIGatewayKey), "the pass stops at the failed DELETE") - require.Len(t, env.fake.aiGatewayKeys(), 2) + require.Len(t, env.fake.aiGatewayKeys(), 3) + // The release log names the keys that the pass listed and left. + require.Contains(t, logs.String(), orphan.ID.String()) } else { require.Empty(t, env.fake.aiGatewayKeys()) } diff --git a/internal/controller/coderaigatewaykey_mismatch_test.go b/internal/controller/coderaigatewaykey_mismatch_test.go index 2e37977d2..77053803a 100644 --- a/internal/controller/coderaigatewaykey_mismatch_test.go +++ b/internal/controller/coderaigatewaykey_mismatch_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/coder/coder/v2/codersdk" + "github.com/google/uuid" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -178,3 +179,37 @@ func TestCoderAIGatewayKeyAnswerMismatchWithFlappingList(t *testing.T) { }) } } + +// A kill-switch re-arm issues a key behind the same gate: a mismatched +// answer to the re-armed create blocks every later create in that +// generation, also across a failed LIST and the settle window. +func TestCoderAIGatewayKeyRearmKeepsAnswerMismatchGate(t *testing.T) { + t.Parallel() + ctx := context.Background() + env := newAIGatewayKeyEnv(ctx, t, ptrTo(true)) + got, _ := env.reconcile(ctx, t, env.createKey(ctx, t, "coder", testAIGatewaySecret)) + requireAIGatewayKeyReady(t, got) + require.NoError(t, env.fake.client(t, 5*time.Second).DeleteAIGatewayKey(ctx, uuid.MustParse(got.Status.CurrentKey.ID))) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, true, "KeyDeletedExternally") + + got.Spec.Rotation.Trigger = "re-arm" + require.NoError(t, k8sClient.Update(ctx, got)) + got, _ = env.reconcile(ctx, t, got) // The re-arm clears the current key. + env.fake.failNext(routeCreateAIGatewayKey, answerMismatch()) + got, _ = env.reconcile(ctx, t, got) + requireMismatchStalled(t, got) + require.Equal(t, got.Generation, got.Status.AnswerMismatchGeneration) + creates := env.fake.requestCount(routeCreateAIGatewayKey) + + env.fake.failNext(routeAIGatewayKeys, fakeFault{Status: http.StatusBadGateway}) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, false, "CoderUnavailable") + for _, step := range []time.Duration{0, 5 * time.Minute, 0} { + env.clock.Step(step) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, true, "CoderAnswerMismatch") + } + require.Equal(t, creates, env.fake.requestCount(routeCreateAIGatewayKey), "no create after the mismatch") + require.Empty(t, env.fake.aiGatewayKeys(), "the sweep deleted the mismatched key") +} diff --git a/internal/controller/coderaigatewaykey_recover.go b/internal/controller/coderaigatewaykey_recover.go index bcd9880af..423cb7ba7 100644 --- a/internal/controller/coderaigatewaykey_recover.go +++ b/internal/controller/coderaigatewaykey_recover.go @@ -2,9 +2,7 @@ package controller import ( "context" - "errors" "fmt" - "net/http" "strings" "time" @@ -293,7 +291,9 @@ func (r *CoderAIGatewayKeyReconciler) deleteOwnKey( if held == k.ID { return aiGatewayKeyWait(time.Second, reason, "Secret %q now holds key %s, so the operator does not delete it.", key.Spec.SecretName, k.Name) } - if err := sdk.DeleteAIGatewayKey(ctx, k.ID); (err != nil && coderStatus(err) != http.StatusNotFound) || coderRouteMissing(err) { + // Only Coder's own 404 proves that the key is gone. An HTML 404 of a proxy + // or "Route not found." proves nothing. + if err := sdk.DeleteAIGatewayKey(ctx, k.ID); err != nil && !isCoderResourceNotFound(err) { failure := aiGatewayKeyCoderFailure("delete "+noun+" "+k.Name, err) if failure.stalled { failure.message = fmt.Sprintf("The operator cannot delete %s %s. %s", noun, k.Name, failure.message) @@ -318,14 +318,6 @@ func aiGatewayKeyKeepReady(key *coderv1alpha1.CoderAIGatewayKey, step *aiGateway return step } -// coderRouteMissing reports Coder's 404 for a route that it does not have -// (plan F11). Unlike the 404 for a missing key, it proves nothing was -// deleted. -func coderRouteMissing(err error) bool { - var sdkErr *codersdk.Error - return errors.As(err, &sdkErr) && sdkErr.StatusCode() == http.StatusNotFound && sdkErr.Message == aiGatewayRouteNotFound -} - // liveSecretHolds live-reads the output Secret and returns the key in own // that it holds (plan I1), whoever owns the Secret: in delete-only mode a // replaced Secret can still hold a key of this object (plan 4.8 rule 2). The diff --git a/internal/controller/coderaigatewaykey_stall_test.go b/internal/controller/coderaigatewaykey_stall_test.go index 78b7c9971..890a0ded3 100644 --- a/internal/controller/coderaigatewaykey_stall_test.go +++ b/internal/controller/coderaigatewaykey_stall_test.go @@ -345,3 +345,41 @@ func TestCoderAIGatewayKeyUndeletableOrphanWhileReady(t *testing.T) { }) } } + +// Only Coder's own 404 proves that a key is gone. An HTML 404 of a proxy on +// a sweep or retire DELETE is CoderUnavailable: the orphan stays an orphan, +// the replaced key stays retiring, and the next DELETE runs. +func TestCoderAIGatewayKeyProxyNotFoundDeletesNothing(t *testing.T) { + t.Parallel() + ctx := context.Background() + proxy404 := fakeFault{Status: http.StatusNotFound, HTML: "404"} + t.Run("sweep", func(t *testing.T) { + t.Parallel() + env := newAIGatewayKeyEnv(ctx, t, ptrTo(true)) + got, orphan := readyWithOrphan(ctx, t, &env) + env.fake.failNext(routeDeleteAIGatewayKey, proxy404) + got, _ = env.reconcile(ctx, t, got) + requireReadyWhile(t, got, "CoderUnavailable") + require.Len(t, env.fake.aiGatewayKeys(), 2) + got, _ = env.reconcile(ctx, t, got) + requireReadyWhile(t, got, "DeletingOrphans") + require.Equal(t, 2, env.fake.requestCount(routeDeleteAIGatewayKey), "the sweep tries again") + for _, k := range env.fake.aiGatewayKeys() { + require.NotEqual(t, orphan.id, k.id) + } + }) + t.Run("retire", func(t *testing.T) { + t.Parallel() + env := newAIGatewayKeyEnv(ctx, t, ptrTo(true)) + got, old := retiringAfterDrift(ctx, t, &env) + env.clock.Step(time.Hour) + env.fake.failNext(routeDeleteAIGatewayKey, proxy404) + got, _ = env.reconcile(ctx, t, got) + requireReadyWhile(t, got, "CoderUnavailable") + require.Equal(t, []string{old.id.String()}, retiringIDs(got), "the key stays retiring") + got, _ = env.reconcile(ctx, t, got) + requireReadyWhile(t, got, "RetiringKey") + require.Empty(t, got.Status.RetiringKeys) + require.Len(t, env.fake.aiGatewayKeys(), 1) + }) +} From 2353e1c02190b6e5917506486a5fec5d57380edf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 07:42:43 +0000 Subject: [PATCH 2/5] feat: activate the CoderAIGatewayKey controller Change-Id: Id9d090afd3e29026d578116a592057b1ab6f007a Signed-off-by: Thomas Kosiewski --- .cspell.json | 2 + .github/workflows/ci.yaml | 19 +- api/v1alpha1/coderaigatewaykey_types.go | 3 - api/v1alpha1/codercontrolplane_types.go | 3 +- .../coder.com_coderaigatewaykeys.yaml | 0 .../bases/coder.com_codercontrolplanes.yaml | 6 +- config/default/kustomization.yaml | 1 + config/rbac/role.yaml | 14 +- .../coder_v1alpha1_coderaigatewaykey.yaml | 24 ++ dist/install.yaml | 403 +++++++++++++++++- docs/explanation/architecture.md | 4 +- docs/how-to/deploy-controller.md | 2 +- docs/how-to/manage-ai-gateway-keys.md | 197 +++++++++ docs/index.md | 2 +- docs/reference/api/coderaigatewaykey.md | 99 +++++ docs/reference/api/codercontrolplane.md | 4 +- docs/tutorials/getting-started.md | 5 +- hack/update-manifests.sh | 2 +- internal/app/controllerapp/controllerapp.go | 10 + .../coderaigatewaykey_controller.go | 16 +- .../coderaigatewaykey_controller_test.go | 40 ++ .../codercontrolplane_aigateway_api_test.go | 4 +- .../controller/controllerapp_setup_test.go | 41 ++ mkdocs.yml | 2 + 24 files changed, 842 insertions(+), 61 deletions(-) rename config/crd/{dormant => bases}/coder.com_coderaigatewaykeys.yaml (100%) create mode 100644 config/samples/coder_v1alpha1_coderaigatewaykey.yaml create mode 100644 docs/how-to/manage-ai-gateway-keys.md create mode 100644 docs/reference/api/coderaigatewaykey.md create mode 100644 internal/controller/controllerapp_setup_test.go diff --git a/.cspell.json b/.cspell.json index a46b009c6..7473cac53 100644 --- a/.cspell.json +++ b/.cspell.json @@ -47,6 +47,8 @@ "codertemplate", "codertemplates", "aibridge", + "coderaigatewaykey", + "coderaigatewaykeys", "coderaiprovider", "coderaiproviders", "codertemplatetest", diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22ba36de0..645ec1401 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,6 +86,9 @@ jobs: - 'api/v1alpha1/**' - 'internal/controller/codertemplatetest*' - 'internal/controller/coderaiprovider*' + - 'internal/controller/coderaigatewaykey*' + - 'internal/controller/owned_secret.go' + - 'internal/controller/reserved_secret_names.go' # Shared Coder access that CoderTemplateTest and later pool controllers call. - 'internal/controller/controlplane_access.go' - 'internal/controller/coder_access.go' @@ -156,22 +159,6 @@ jobs: exit 1 fi - - name: Verify the dormant CoderAIGatewayKey CRD is not installed - # Remove this step when the CoderAIGatewayKey controller ships (#156). - run: | - # grep exits 1 when nothing matches. Any other status (for example a missing file) fails the step. - status=0 - # The third pattern matches the CRD path entry in config/default/kustomization.yaml. - grep -n -E '^[[:space:]]+kind: CoderAIGatewayKey$|coderaigatewaykeys\.coder\.com|coder\.com_coderaigatewaykeys\.yaml' \ - dist/install.yaml config/default/kustomization.yaml config/crd/bases/*.yaml || status=$? - if [[ "${status}" -eq 0 ]]; then - echo "assertion failed: the CoderAIGatewayKey CRD must not ship before its controller" >&2 - exit 1 - elif [[ "${status}" -ne 1 ]]; then - echo "assertion failed: the CoderAIGatewayKey CRD guard could not read the install manifests (grep exit ${status})" >&2 - exit 1 - fi - - name: Verify the dormant CoderMCPServer CRD is not installed # Remove this step when the CoderMCPServer controller ships (#155). run: | diff --git a/api/v1alpha1/coderaigatewaykey_types.go b/api/v1alpha1/coderaigatewaykey_types.go index 5a09d63d8..1f3051ca4 100644 --- a/api/v1alpha1/coderaigatewaykey_types.go +++ b/api/v1alpha1/coderaigatewaykey_types.go @@ -36,9 +36,6 @@ type AIGatewaySpec struct { // controller creates no key. Setting it back to false stops new keys and // rotations. The current key of each CoderAIGatewayKey stays until that // object is deleted. - // - // This field has no effect yet: the CoderAIGatewayKey controller is not - // shipped. // +optional AllowKeys bool `json:"allowKeys,omitempty"` } diff --git a/api/v1alpha1/codercontrolplane_types.go b/api/v1alpha1/codercontrolplane_types.go index 3f20088ea..13c2cfae7 100644 --- a/api/v1alpha1/codercontrolplane_types.go +++ b/api/v1alpha1/codercontrolplane_types.go @@ -132,8 +132,7 @@ type CoderControlPlaneSpec struct { MCPServers *MCPServersSpec `json:"mcpServers,omitempty"` // AIGateway configures CoderAIGatewayKey objects that use this control - // plane. It has no effect yet: the CoderAIGatewayKey controller is not - // shipped. + // plane. // +optional AIGateway *AIGatewaySpec `json:"aiGateway,omitempty"` diff --git a/config/crd/dormant/coder.com_coderaigatewaykeys.yaml b/config/crd/bases/coder.com_coderaigatewaykeys.yaml similarity index 100% rename from config/crd/dormant/coder.com_coderaigatewaykeys.yaml rename to config/crd/bases/coder.com_coderaigatewaykeys.yaml diff --git a/config/crd/bases/coder.com_codercontrolplanes.yaml b/config/crd/bases/coder.com_codercontrolplanes.yaml index 87d0c56cd..a0953b7c7 100644 --- a/config/crd/bases/coder.com_codercontrolplanes.yaml +++ b/config/crd/bases/coder.com_codercontrolplanes.yaml @@ -954,8 +954,7 @@ spec: aiGateway: description: |- AIGateway configures CoderAIGatewayKey objects that use this control - plane. It has no effect yet: the CoderAIGatewayKey controller is not - shipped. + plane. properties: allowKeys: description: |- @@ -967,9 +966,6 @@ spec: controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. - - This field has no effect yet: the CoderAIGatewayKey controller is not - shipped. type: boolean type: object certs: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index a2042c150..d3f9b0e3a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace-coder-system.yaml + - ../crd/bases/coder.com_coderaigatewaykeys.yaml - ../crd/bases/coder.com_coderaiproviders.yaml - ../crd/bases/coder.com_codercontrolplanes.yaml - ../crd/bases/coder.com_coderprovisioners.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f1d9c5589..561f059eb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -85,12 +85,6 @@ rules: - coder.com resources: - coderaigatewaykeys - - codermcpservers - verbs: - - list -- apiGroups: - - coder.com - resources: - coderaiproviders verbs: - get @@ -100,6 +94,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/finalizers - coderaiproviders/finalizers - codercontrolplanes/finalizers - coderprovisioners/finalizers @@ -110,6 +105,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/status - coderaiproviders/status verbs: - get @@ -139,6 +135,12 @@ rules: - get - patch - update +- apiGroups: + - coder.com + resources: + - codermcpservers + verbs: + - list - apiGroups: - coder.com resources: diff --git a/config/samples/coder_v1alpha1_coderaigatewaykey.yaml b/config/samples/coder_v1alpha1_coderaigatewaykey.yaml new file mode 100644 index 000000000..67b859e3a --- /dev/null +++ b/config/samples/coder_v1alpha1_coderaigatewaykey.yaml @@ -0,0 +1,24 @@ +# Keeps one Coder AI Gateway key in an owned Secret and rotates it. See +# docs/how-to/manage-ai-gateway-keys.md. +# +# Prerequisites: +# - A Coder Premium license with the AI Gateway enabled. +# - CoderControlPlane codercontrolplane-sample with status.operatorAccessReady +# true and spec.aiGateway.allowKeys true. +# - No Secret named coder-ai-gateway-key in the namespace: the operator +# creates it. +apiVersion: coder.com/v1alpha1 +kind: CoderAIGatewayKey +metadata: + name: gateway + namespace: coder +spec: + # Immutable. + controlPlaneRef: + name: codercontrolplane-sample + # Immutable. The key goes into data key "key". + secretName: coder-ai-gateway-key + rotation: + intervalSeconds: 2592000 + gracePeriodSeconds: 3600 + consumerWaitSeconds: 86400 diff --git a/dist/install.yaml b/dist/install.yaml index b8e7b64ea..9a7637b0d 100644 --- a/dist/install.yaml +++ b/dist/install.yaml @@ -5,6 +5,389 @@ metadata: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: coderaigatewaykeys.coder.com +spec: + group: coder.com + names: + kind: CoderAIGatewayKey + listKind: CoderAIGatewayKeyList + plural: coderaigatewaykeys + singular: coderaigatewaykey + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.secretName + name: Secret + type: string + - jsonPath: .status.currentKey.name + name: Current + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.nextRotationTime + name: Next rotation + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway + replicas and stores it in a Secret that this object owns. A gateway key + lets its holder load the API key of every enabled AI provider, so write + access to this kind is admin-only. Deleting the object revokes every key + that it created at once, and gateways that use them stop within a minute. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CoderAIGatewayKeySpec defines a Coder AI Gateway key and + its rotation. + properties: + controlPlaneRef: + description: |- + ControlPlaneRef names the CoderControlPlane in the same namespace. The + controller creates and deletes keys with that control plane's operator + token. The control plane must allow keys in spec.aiGateway.allowKeys. + properties: + name: + description: Name is the CoderControlPlane name. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: controlPlaneRef is immutable + rule: self == oldSelf + rotation: + default: {} + description: |- + Rotation configures when the controller replaces the key and how long a + replaced key stays valid. + properties: + consumerWaitSeconds: + default: 86400 + description: |- + ConsumerWaitSeconds is how long after the grace period the controller + keeps a replaced key while Coder still reports connections that use it. + After that time the controller deletes the key, and gateways that still + use it exit and restart with the key from the Secret. 0 deletes the key + when the grace period ends. At most 7 days, default 1 day. + format: int32 + maximum: 604800 + minimum: 0 + type: integer + gracePeriodSeconds: + default: 3600 + description: |- + GracePeriodSeconds is the minimum time between writing a new key to the + Secret and deleting the replaced key in Coder. Consumers must reload + within it. Between 10 minutes and 7 days, default 1 hour. + format: int32 + maximum: 604800 + minimum: 600 + type: integer + intervalSeconds: + default: 2592000 + description: |- + IntervalSeconds is the time between scheduled rotations, counted from + the time the current key was written to the Secret. 0 turns scheduled + rotation off. Otherwise it is between 1 day and 365 days. The default is + 30 days, because Coder keys never expire. + format: int32 + maximum: 31536000 + minimum: 0 + type: integer + x-kubernetes-validations: + - message: intervalSeconds must be 0 or at least 86400 (1 day) + rule: self == 0 || self >= 86400 + trigger: + description: |- + Trigger starts one rotation each time it changes to a new non-empty + value. The value set when the object is created starts no rotation. + maxLength: 63 + type: string + type: object + x-kubernetes-validations: + - message: gracePeriodSeconds must be shorter than intervalSeconds + rule: '!has(self.intervalSeconds) || !has(self.gracePeriodSeconds) + || self.intervalSeconds == 0 || self.gracePeriodSeconds < self.intervalSeconds' + secretName: + description: |- + SecretName is the Secret in the same namespace that receives the key, + under the data key "key". The controller creates the Secret and owns it. + It never writes a Secret that already exists and that this object does + not own. To move the key to another Secret, delete and re-create this + object. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - controlPlaneRef + - secretName + type: object + status: + description: |- + CoderAIGatewayKeyStatus defines the observed state of a CoderAIGatewayKey. + It holds Coder key IDs, names, serials, and times only. It never holds a + key, a key prefix, a hash, or a masked key. + properties: + answerMismatchGeneration: + description: |- + AnswerMismatchGeneration is the generation in which Coder answered a + create with a key that did not match the request. The controller + creates no key until the generation changes. 0 means no mismatch. + format: int64 + minimum: 0 + type: integer + appliedGeneration: + description: |- + AppliedGeneration is the last generation that the controller fully + applied. A spec change re-arms a stall that waits for one. + format: int64 + type: integer + conditions: + description: Conditions are Ready, Reconciling, and Stalled. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + consecutiveIssueFailures: + description: |- + ConsecutiveIssueFailures counts create requests in a row whose outcome + was uncertain, and failed Secret writes. A success resets it. + format: int32 + minimum: 0 + type: integer + currentKey: + description: CurrentKey is the key that the output Secret holds. + properties: + id: + description: ID is the Coder key ID. It is not the key. + format: uuid + maxLength: 36 + type: string + issuedAt: + description: IssuedAt is when the controller wrote the key to + the Secret. + format: date-time + type: string + name: + description: Name is the Coder key name. + maxLength: 64 + type: string + serial: + description: Serial is the key serial. + format: int64 + minimum: 0 + type: integer + required: + - id + - issuedAt + - name + - serial + type: object + killSwitchGeneration: + description: |- + KillSwitchGeneration is the generation in which the controller found + the current key deleted in Coder. The controller creates no key until + the generation changes. 0 means the kill switch is not active. + format: int64 + minimum: 0 + type: integer + lastCreateAttemptTime: + description: |- + LastCreateAttemptTime is when the controller last sent a key create + request. Orphaned keys are deleted only after a settle window from this + time. + format: date-time + type: string + lastDriftReissueTime: + description: |- + LastDriftReissueTime is when the controller last issued a key because + the Secret no longer held the current key. + format: date-time + type: string + lastHandledTrigger: + description: |- + LastHandledTrigger is the spec.rotation.trigger value that the + controller last acted on. + maxLength: 63 + type: string + nextRotationTime: + description: NextRotationTime is when the next scheduled rotation + is due. + format: date-time + type: string + nextSerial: + description: |- + NextSerial is the serial of the next key create request. It never + decreases, so the controller never reuses a key name. + format: int64 + minimum: 0 + type: integer + observedGeneration: + description: ObservedGeneration is the generation the controller last + processed. + format: int64 + type: integer + pendingKey: + description: |- + PendingKey is set before every key create request and cleared after it. + While it is set, a create request can be in flight or already committed + in Coder. + properties: + createAttemptTime: + description: CreateAttemptTime is when the controller wrote this + marker. + format: date-time + type: string + name: + description: Name is the Coder key name of the create request. + maxLength: 64 + type: string + serial: + description: Serial is the key serial of the create request. + format: int64 + minimum: 0 + type: integer + required: + - createAttemptTime + - name + - serial + type: object + retiringKeys: + description: RetiringKeys are replaced keys that wait for deletion + in Coder. + items: + description: |- + CoderAIGatewayKeyRetiring records a replaced key that waits for deletion in + Coder. It holds no secret data and nothing derived from it. + properties: + id: + description: ID is the Coder key ID. It is not the key. + format: uuid + maxLength: 36 + type: string + name: + description: Name is the Coder key name. + maxLength: 64 + type: string + replacedAt: + description: |- + ReplacedAt is when a newer key replaced this key in the Secret. The + grace period counts from this time. + format: date-time + type: string + serial: + description: Serial is the key serial. + format: int64 + minimum: 0 + type: integer + required: + - id + - name + - replacedAt + - serial + type: object + maxItems: 3 + type: array + x-kubernetes-list-map-keys: + - serial + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 @@ -1229,8 +1612,7 @@ spec: aiGateway: description: |- AIGateway configures CoderAIGatewayKey objects that use this control - plane. It has no effect yet: the CoderAIGatewayKey controller is not - shipped. + plane. properties: allowKeys: description: |- @@ -1242,9 +1624,6 @@ spec: controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. - - This field has no effect yet: the CoderAIGatewayKey controller is not - shipped. type: boolean type: object certs: @@ -6107,12 +6486,6 @@ rules: - coder.com resources: - coderaigatewaykeys - - codermcpservers - verbs: - - list -- apiGroups: - - coder.com - resources: - coderaiproviders verbs: - get @@ -6122,6 +6495,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/finalizers - coderaiproviders/finalizers - codercontrolplanes/finalizers - coderprovisioners/finalizers @@ -6132,6 +6506,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/status - coderaiproviders/status verbs: - get @@ -6161,6 +6536,12 @@ rules: - get - patch - update +- apiGroups: + - coder.com + resources: + - codermcpservers + verbs: + - list - apiGroups: - coder.com resources: diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 75a4d256c..c6a02ff4f 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -31,7 +31,7 @@ graph TD | --- | --- | --- | --- | | Code | `internal/app/controllerapp/`, `internal/controller/` | `internal/app/apiserverapp/`, `internal/aggregated/storage/`, `internal/aggregated/coder/` | `internal/app/mcpapp/` | | Listens on | `:8081` (`/healthz`, `/readyz`) | `:6443` HTTPS (default) | `127.0.0.1:8090` only. `/mcp` requires the bearer token. `/healthz` and `/readyz` do not. | -| Resources | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider` | `coderworkspaces`, `codertemplates`, `codertemplateversions` | Tools for control planes, templates, workspaces, events, pod logs, and run state | +| Resources | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, `CoderAIGatewayKey` | `coderworkspaces`, `codertemplates`, `codertemplateversions` | Tools for control planes, templates, workspaces, events, pod logs, and run state | ### Controller @@ -75,7 +75,7 @@ graph TD | Path | Contents | | --- | --- | -| `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, and `CoderAIProvider` | +| `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, and `CoderAIGatewayKey` | | `config/rbac/` | The `coder-k8s` ServiceAccount, the `manager-role` ClusterRole, and the bindings (including auth-delegator) | | `config/apiserver-standalone/` | For standalone `--app=aggregated-apiserver`: the `coder-k8s-apiserver` ServiceAccount, its RBAC, and the placeholder Secret for the serving CA | | `deploy/deployment.yaml` | The `coder-k8s` Deployment (default `--app=all`) | diff --git a/docs/how-to/deploy-controller.md b/docs/how-to/deploy-controller.md index d83ac64f3..ee0691edd 100644 --- a/docs/how-to/deploy-controller.md +++ b/docs/how-to/deploy-controller.md @@ -1,6 +1,6 @@ # Deploy the controller -Run `coder-k8s` as an operator only (`--app=controller`). The operator reconciles `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, and `CoderAIProvider` resources. +Run `coder-k8s` as an operator only (`--app=controller`). The operator reconciles `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, and `CoderAIGatewayKey` resources. Run the commands from a clone of this repository. diff --git a/docs/how-to/manage-ai-gateway-keys.md b/docs/how-to/manage-ai-gateway-keys.md new file mode 100644 index 000000000..8f2e1c378 --- /dev/null +++ b/docs/how-to/manage-ai-gateway-keys.md @@ -0,0 +1,197 @@ +# Manage AI Gateway keys with CoderAIGatewayKey + +A `CoderAIGatewayKey` keeps one Coder AI Gateway key in a Secret. The operator creates the key in Coder (Coder's `/api/v2/ai-gateway/keys`), writes it into a Secret that the object owns, rotates it, deletes replaced keys after a grace period, and deletes every key of the object when you delete the object. A standalone AI Gateway reads the key from the Secret. + +!!! warning "A gateway key is a full-trust credential" + Coder trusts every holder of a gateway key. The holder can load the API keys of every enabled AI provider, read the external-auth tokens that users gave to MCP servers, and act as any user on the gateway. Treat the output Secret like the operator token Secret. Grant `create`, `update`, `patch` and `delete` on `coderaigatewaykeys`, and `get` on the output Secrets, only to principals who may hold that power. + +## Before you start + +- A Coder Premium license with the AI Gateway enabled (`CODER_AI_GATEWAY_ENABLED`). Without it, Coder answers `403` and the object stalls with `LicenseRequired`. +- A `CoderControlPlane` in the same namespace with `status.operatorAccessReady: true`. The operator user must be a site owner. +- Node clocks within 2 minutes of each other, and the API server's default request timeout of 60 seconds (`--request-timeout`). The operator waits a 5-minute settle window after each key create before it treats a key as an orphan. That window holds only under these two conditions. With a longer request timeout, a late create can commit after the window, and its key then stays in Coder. + +## 1. Allow keys on the control plane + +The control plane admin opts in. Without this, the operator creates no key and the object stalls with `NotAllowed`. An object that never created a key makes no Coder call. + +```bash +kubectl -n coder patch codercontrolplane coder --type=merge -p '{"spec":{"aiGateway":{"allowKeys":true}}}' +``` + +This delegates the power of a gateway key to everyone who can create `CoderAIGatewayKey` objects in the namespace. Setting `allowKeys` back to `false` stops new keys and rotations. The current key of each object stays in its Secret and in Coder until you delete the object. + +## 2. Create the key + +```yaml +apiVersion: coder.com/v1alpha1 +kind: CoderAIGatewayKey +metadata: + name: gateway + namespace: coder +spec: + controlPlaneRef: + name: coder + secretName: coder-ai-gateway-key + rotation: + intervalSeconds: 2592000 # 30 days (default). 0 turns scheduled rotation off. + gracePeriodSeconds: 3600 # 1 hour (default), between 10 minutes and 7 days. + consumerWaitSeconds: 86400 # 1 day (default), at most 7 days. +``` + +```bash +kubectl apply -f gateway-key.yaml +kubectl -n coder wait --for=condition=Ready coderaigatewaykey/gateway --timeout=2m +``` + +The sample is in `config/samples/coder_v1alpha1_coderaigatewaykey.yaml`. `spec.controlPlaneRef` and `spec.secretName` cannot change. To move the key to another Secret, delete the object and create a new one. + +The operator creates the Secret and owns it. It holds one data key, `key`, which is the default of the AI Gateway Helm chart. The annotations `coder.com/ai-gateway-key-id` and `coder.com/ai-gateway-key-name` name the current key and change on each rotation. The operator keeps every other label and annotation, for example reloader annotations. + +The operator never writes a Secret that it does not own. An existing Secret with that name stalls the object with `SecretNotOwned`, and the Secret stays unchanged. Names that a control plane or another Coder resource in the namespace uses stall with `SecretNameReserved`. + +The key names in Coder are `ck8s--`, where `` is the object's `metadata.uid` without dashes. Status holds key IDs, names and times only, never the key or its prefix. + +## 3. Use the key in a gateway + +The gateway reads the key once, at startup. A new key in the Secret takes effect only when the gateway pods restart. Set these in the AI Gateway Helm chart: + +- `aigateway.keySecret.name` to `spec.secretName`. +- A reloader, such as Stakater Reloader, that restarts the pods when the Secret changes, for example the pod annotation `secret.reloader.stakater.com/reload: coder-ai-gateway-key`. +- At least 2 replicas, so a restart causes no gap. + +Without a reloader, run `kubectl rollout restart` on the gateway Deployment after each rotation. Without either, the operator deletes the old key when `consumerWaitSeconds` ends. Each old pod then fails its next connection, exits and restarts with the new key, so each replica has a short outage. + +## 4. Read the result + +```bash +kubectl -n coder get coderaigatewaykeys +kubectl -n coder get coderaigatewaykey gateway -o jsonpath='{range .status.conditions[*]}{.type}={.status} {.reason}: {.message}{"\n"}{end}' +``` + +The object has the conditions `Ready`, `Reconciling` and `Stalled`. `Ready=True` means that the Secret holds the current key and Coder lists it. A rotation keeps `Ready=True`, because the Secret stays valid all the time. A cleanup failure next to a valid Secret shows `Ready=True` together with `Stalled=True`, and its message starts with "The key in Secret ... still works." + +| Reason | Meaning | +| --- | --- | +| `Ready` | The Secret holds the current key, and Coder lists it. | +| `IssuingKey` | The operator creates a key, or retries a create whose answer was uncertain. | +| `WaitingForGracePeriod`, `WaitingForConsumers`, `RetiringKey` | A replaced key waits for deletion. See [Rotation](#rotation). | +| `RotationBlocked` | A rotation is due, but replaced keys still wait for deletion. | +| `DeletingOrphans` | A key of this object that no status names waits for the settle window, then the operator deletes it. | +| `ControlPlaneNotReady`, `OperatorAccessNotReady` | The control plane cannot serve Coder calls yet. | +| `CoderUnavailable` | A Coder call failed. The operator retries every 30 seconds. | +| `NotAllowed` | The control plane does not allow keys. See [step 1](#1-allow-keys-on-the-control-plane). | +| `LicenseRequired`, `CoderRejected`, `NotSupported` | Coder refused: no Premium license or AI Gateway, an operator user that is not a site owner, or a Coder version without the key API. | +| `SecretNotOwned`, `SecretNameReserved`, `SecretWriteRejected` | The output Secret cannot be used. Choose another `secretName` (delete and re-create the object), or fix the admission policy that rejects the write. | +| `KeyDeletedExternally` | Someone deleted the current key in Coder. See [Kill switch](#kill-switch). | +| `SecretConflict` | The Secret lost the key again soon after a re-issue. See [Secret drift](#secret-drift). | +| `IssueFailing`, `CoderAnswerMismatch` | Key creates failed 10 times in a row, or Coder answered a create with a key that does not match the request. See [Stalls](#stalls). | +| `DeletingKeys` | The object is being deleted. See [Delete a key](#delete-a-key). | + +## Rotation + +A scheduled rotation is due `intervalSeconds` after the current key was written (`status.nextRotationTime`). A requested rotation starts each time `spec.rotation.trigger` changes to a new non-empty value: + +```bash +kubectl -n coder patch coderaigatewaykey gateway --type=merge -p '{"spec":{"rotation":{"trigger":"2026-10-14"}}}' +``` + +A rotation creates a new key, writes it into the Secret, and keeps the replaced key in `status.retiringKeys`. The operator deletes a replaced key after `gracePeriodSeconds`, once Coder shows that no gateway uses it: it has no heartbeat for 10 minutes, or the new key's heartbeat is newer. While a gateway still uses the old key, the object shows `WaitingForConsumers`: restart the gateway pods. The operator deletes the old key when `consumerWaitSeconds` ends anyway. + +At most 3 replaced keys wait at a time. A scheduled rotation starts only while no key waits, and a requested rotation only while fewer than 2 wait. Otherwise the object shows `RotationBlocked` and the rotation starts when a slot frees. + +## Secret drift + +When the Secret loses the current key (someone edits or deletes it), the operator creates a new key, writes it, and lets the old key retire with the full grace period. When the Secret loses the key again within the grace period, the object stalls with `SecretConflict`: another writer, such as a GitOps tool, likely manages the same Secret. Remove the other writer. The stall ends when the grace period after the last re-issue ends. With 3 replaced keys waiting, it ends when the operator deletes one of them. + +## Kill switch + +Delete the current key in Coder to revoke it at once. Gateways that use it lose their connection within 60 seconds. + +```bash +coder ai-gateway keys list +coder ai-gateway keys delete +``` + +The object then stalls with `KeyDeletedExternally`, and the operator creates no key until the spec changes. Edit the spec, for example change `spec.rotation.trigger`, to issue a new key. Two cases need a second spec edit: a spec change made before the operator saw the deleted key, and a spec edit that ended a `SecretNotOwned`, `SecretNameReserved` or `NotAllowed` stall while the key was already deleted. + +## Stalls + +A stalled object creates no key until its spec changes. It still deletes orphans and replaced keys. + +- `IssueFailing`: 10 key creates in a row had an uncertain answer, or their Secret write failed. Fix the cause in the message, then edit the spec. +- `CoderAnswerMismatch`: Coder answered a create with a key that does not match the request. The created key is an orphan, and the operator deletes it after the settle window. Check the Coder version and any proxy between the operator and Coder, then edit the spec. + +## Delete a key + +Deleting the object deletes every key of the object in Coder, the current key included. Gateways that use them lose access within 60 seconds. The Secret goes with the object. + +1. The operator waits until 5 minutes after its last key create, so that a create still in flight commits first (`DeletingKeys`). +2. It lists the keys, deletes each key of this object, and removes the finalizer `coder.com/ai-gateway-key-cleanup` only after a list shows none. + +The finalizer stays while Coder cannot confirm the deletion: + +| Condition | Retry | +| --- | --- | +| `CoderUnavailable`: Coder does not answer, answers `5xx` or `429`, or answers `404` that is not Coder's own (for example an HTML page from a proxy) | Every 60 seconds, with no deadline | +| `ControlPlaneNotReady`, `OperatorAccessNotReady` | Every 60 seconds | +| `LicenseRequired`, `CoderRejected`, `NotSupported` (`Stalled=True`) | Every 10 minutes | + +When the control plane is gone, nothing can delete the keys. The finalizer goes at once. The operator logs `Releasing the finalizer without deleting the keys in Coder` with reason `ControlPlaneGone` and the key IDs: delete those keys with `coder ai-gateway keys delete`. + +While the control plane is being deleted, as in a namespace deletion, the operator runs one pass of at most 10 seconds without the settle window, then removes the finalizer whatever the answer. A namespace deletion therefore finishes. A key create that was still in flight at that moment can commit later, and its key stays in Coder. When the pass fails, the log lists the IDs that it did not delete. + +Delete `CoderAIGatewayKey` objects before their control plane. + +### Stuck finalizer + +A deletion that keeps failing never ends by itself. Leave it in one of these ways: + +1. **Fix Coder or the operator user.** Restore Coder access, the Premium license or the AI Gateway, the operator user's site-owner role, or the proxy that answers `404`. The next retry deletes the keys and releases the object. +2. **Delete the control plane.** The operator then runs its one bounded pass and releases the object. Do this only when you remove that Coder deployment anyway. +3. **Remove the finalizer by hand, then delete the keys in Coder.** + + Note the key names from `status.currentKey` and `status.retiringKeys`, and the object's UID (`kubectl -n coder get coderaigatewaykey gateway -o jsonpath='{.metadata.uid}'`). Find the index of the finalizer: + + ```bash + kubectl -n coder get coderaigatewaykey gateway -o jsonpath='{.metadata.finalizers}{"\n"}' + ``` + + Remove it with a JSON patch. The `test` operation makes the patch fail, with no change, if another finalizer is at that index. Replace `0` with the index: + + ```bash + kubectl -n coder patch coderaigatewaykey gateway --type=json -p '[ + {"op": "test", "path": "/metadata/finalizers/0", "value": "coder.com/ai-gateway-key-cleanup"}, + {"op": "remove", "path": "/metadata/finalizers/0"} + ]' + ``` + + Then delete every key whose name starts with `ck8s--` with `coder ai-gateway keys delete `. + +## Known limits + +- **Leftover keys.** A key that commits more than 5 minutes after a lost create answer, or after the object is gone, stays in Coder. Find keys of deleted objects with `coder ai-gateway keys list`: their names start with `ck8s-` and a UID that no `CoderAIGatewayKey` has. +- **A short key list.** The operator reads all keys with one list call. When a list answer names the current key but leaves out a replaced key, the operator treats the replaced key as deleted. If the key appears again, the operator deletes it as an orphan, without the rest of its grace period. Coder lists all keys in one answer, so only a proxy between the operator and Coder that changes the answer can cause this. + +## Grant access + +| Permission | Coder power | +| --- | --- | +| `create`, `update`, `patch` on `coderaigatewaykeys` (with `allowKeys: true`) | Mint full-trust gateway keys with the operator token | +| `delete` on `coderaigatewaykeys` | Revokes the object's keys for every gateway that uses them | +| `get` on the output Secret | Holds a full-trust gateway key | +| `get`, `list` on `coderaigatewaykeys` | Sees Secret names, key names, Coder key IDs and times. No secrets. | + +A viewer Role: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: coderaigatewaykey-viewer + namespace: coder +rules: + - apiGroups: ["coder.com"] + resources: ["coderaigatewaykeys"] + verbs: ["get", "list", "watch"] +``` diff --git a/docs/index.md b/docs/index.md index c82d1a7e7..9016006f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Use native Kubernetes APIs to run and manage [Coder](https://coder.com). | Component | Resources | | --- | --- | -| Operator | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider` (`coder.com/v1alpha1`) | +| Operator | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, `CoderAIGatewayKey` (`coder.com/v1alpha1`) | | Aggregated API server | `CoderWorkspace`, `CoderTemplate`, `CoderTemplateVersion` (`aggregation.coder.com/v1alpha1`) | | MCP server | Operational tools over HTTP | diff --git a/docs/reference/api/coderaigatewaykey.md b/docs/reference/api/coderaigatewaykey.md new file mode 100644 index 000000000..8d16fb076 --- /dev/null +++ b/docs/reference/api/coderaigatewaykey.md @@ -0,0 +1,99 @@ + + +# `CoderAIGatewayKey` + +## API identity + +- Group/version: `coder.com/v1alpha1` +- Kind: `CoderAIGatewayKey` +- Resource: `coderaigatewaykeys` +- Scope: namespaced + +## Spec + +| Field | Type | Description | +| --- | --- | --- | +| `controlPlaneRef` | [CoderControlPlaneReference](#codercontrolplanereference) | ControlPlaneRef names the CoderControlPlane in the same namespace. The controller creates and deletes keys with that control plane's operator token. The control plane must allow keys in spec.aiGateway.allowKeys. | +| `secretName` | string | SecretName is the Secret in the same namespace that receives the key, under the data key "key". The controller creates the Secret and owns it. It never writes a Secret that already exists and that this object does not own. To move the key to another Secret, delete and re-create this object. | +| `rotation` | [CoderAIGatewayKeyRotationSpec](#coderaigatewaykeyrotationspec) | Rotation configures when the controller replaces the key and how long a replaced key stays valid. | + +## Status + +| Field | Type | Description | +| --- | --- | --- | +| `observedGeneration` | integer | ObservedGeneration is the generation the controller last processed. | +| `appliedGeneration` | integer | AppliedGeneration is the last generation that the controller fully applied. A spec change re-arms a stall that waits for one. | +| `nextSerial` | integer | NextSerial is the serial of the next key create request. It never decreases, so the controller never reuses a key name. | +| `lastCreateAttemptTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | LastCreateAttemptTime is when the controller last sent a key create request. Orphaned keys are deleted only after a settle window from this time. | +| `pendingKey` | [CoderAIGatewayKeyPending](#coderaigatewaykeypending) | PendingKey is set before every key create request and cleared after it. While it is set, a create request can be in flight or already committed in Coder. | +| `currentKey` | [CoderAIGatewayKeyCurrent](#coderaigatewaykeycurrent) | CurrentKey is the key that the output Secret holds. | +| `retiringKeys` | [CoderAIGatewayKeyRetiring](#coderaigatewaykeyretiring) array | RetiringKeys are replaced keys that wait for deletion in Coder. | +| `lastHandledTrigger` | string | LastHandledTrigger is the spec.rotation.trigger value that the controller last acted on. | +| `nextRotationTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | NextRotationTime is when the next scheduled rotation is due. | +| `consecutiveIssueFailures` | integer | ConsecutiveIssueFailures counts create requests in a row whose outcome was uncertain, and failed Secret writes. A success resets it. | +| `lastDriftReissueTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | LastDriftReissueTime is when the controller last issued a key because the Secret no longer held the current key. | +| `killSwitchGeneration` | integer | KillSwitchGeneration is the generation in which the controller found the current key deleted in Coder. The controller creates no key until the generation changes. 0 means the kill switch is not active. | +| `answerMismatchGeneration` | integer | AnswerMismatchGeneration is the generation in which Coder answered a create with a key that did not match the request. The controller creates no key until the generation changes. 0 means no mismatch. | +| `conditions` | [Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array | Conditions are Ready, Reconciling, and Stalled. | + +## Referenced types + +### CoderAIGatewayKeyCurrent + +CoderAIGatewayKeyCurrent records the key that the output Secret holds. It +holds no secret data and nothing derived from it. + +| Field | Type | Description | +| --- | --- | --- | +| `serial` | integer | Serial is the key serial. | +| `id` | string | ID is the Coder key ID. It is not the key. | +| `name` | string | Name is the Coder key name. | +| `issuedAt` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | IssuedAt is when the controller wrote the key to the Secret. | + +### CoderAIGatewayKeyPending + +CoderAIGatewayKeyPending is the create marker. The controller writes it +before every key create request. It holds no secret data. + +| Field | Type | Description | +| --- | --- | --- | +| `serial` | integer | Serial is the key serial of the create request. | +| `name` | string | Name is the Coder key name of the create request. | +| `createAttemptTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | CreateAttemptTime is when the controller wrote this marker. | + +### CoderAIGatewayKeyRetiring + +CoderAIGatewayKeyRetiring records a replaced key that waits for deletion in +Coder. It holds no secret data and nothing derived from it. + +| Field | Type | Description | +| --- | --- | --- | +| `serial` | integer | Serial is the key serial. | +| `id` | string | ID is the Coder key ID. It is not the key. | +| `name` | string | Name is the Coder key name. | +| `replacedAt` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | ReplacedAt is when a newer key replaced this key in the Secret. The grace period counts from this time. | + +### CoderAIGatewayKeyRotationSpec + +CoderAIGatewayKeyRotationSpec configures key rotation. ++kubebuilder:validation:XValidation:rule="!has(self.intervalSeconds) || !has(self.gracePeriodSeconds) || self.intervalSeconds == 0 || self.gracePeriodSeconds < self.intervalSeconds",message="gracePeriodSeconds must be shorter than intervalSeconds" + +| Field | Type | Description | +| --- | --- | --- | +| `intervalSeconds` | integer | IntervalSeconds is the time between scheduled rotations, counted from the time the current key was written to the Secret. 0 turns scheduled rotation off. Otherwise it is between 1 day and 365 days. The default is 30 days, because Coder keys never expire. | +| `gracePeriodSeconds` | integer | GracePeriodSeconds is the minimum time between writing a new key to the Secret and deleting the replaced key in Coder. Consumers must reload within it. Between 10 minutes and 7 days, default 1 hour. | +| `consumerWaitSeconds` | integer | ConsumerWaitSeconds is how long after the grace period the controller keeps a replaced key while Coder still reports connections that use it. After that time the controller deletes the key, and gateways that still use it exit and restart with the key from the Secret. 0 deletes the key when the grace period ends. At most 7 days, default 1 day. | +| `trigger` | string | Trigger starts one rotation each time it changes to a new non-empty value. The value set when the object is created starts no rotation. | + +### CoderControlPlaneReference + +CoderControlPlaneReference names a CoderControlPlane in the same namespace. + +| Field | Type | Description | +| --- | --- | --- | +| `name` | string | Name is the CoderControlPlane name. | + +## Source + +- Go type: `api/v1alpha1/coderaigatewaykey_types.go` +- Generated CRD: `config/crd/bases/coder.com_coderaigatewaykeys.yaml` diff --git a/docs/reference/api/codercontrolplane.md b/docs/reference/api/codercontrolplane.md index ab5aecd7b..1a5da6f3e 100644 --- a/docs/reference/api/codercontrolplane.md +++ b/docs/reference/api/codercontrolplane.md @@ -41,7 +41,7 @@ | `affinity` | [Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#affinity-v1-core) | Affinity configures pod affinity/anti-affinity rules. | | `topologySpreadConstraints` | [TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#topologyspreadconstraint-v1-core) array | TopologySpreadConstraints control pod topology spread. | | `mcpServers` | [MCPServersSpec](#mcpserversspec) | MCPServers configures CoderMCPServer objects that use this control plane. It has no effect yet: the CoderMCPServer controller is not shipped. | -| `aiGateway` | [AIGatewaySpec](#aigatewayspec) | AIGateway configures CoderAIGatewayKey objects that use this control plane. It has no effect yet: the CoderAIGatewayKey controller is not shipped. | +| `aiGateway` | [AIGatewaySpec](#aigatewayspec) | AIGateway configures CoderAIGatewayKey objects that use this control plane. | | `workspacePools` | [WorkspacePoolsSpec](#workspacepoolsspec) | WorkspacePools configures CoderWorkspacePool objects that use this control plane. It has no effect yet: the CoderWorkspacePool controller is not shipped. | | `templateTests` | [TemplateTestsSpec](#templatetestsspec) | TemplateTests configures CoderTemplateTest runs against this control plane. | @@ -71,7 +71,7 @@ CoderControlPlane (CoderControlPlane.spec.aiGateway). | Field | Type | Description | | --- | --- | --- | -| `allowKeys` | boolean | AllowKeys lets CoderAIGatewayKey objects in this namespace create and rotate Coder AI Gateway keys with this control plane's operator token. A gateway key lets its holder load the API key of every enabled AI provider, so this delegates that power to everyone who can create CoderAIGatewayKey objects here. When false (the default), the controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. This field has no effect yet: the CoderAIGatewayKey controller is not shipped. | +| `allowKeys` | boolean | AllowKeys lets CoderAIGatewayKey objects in this namespace create and rotate Coder AI Gateway keys with this control plane's operator token. A gateway key lets its holder load the API key of every enabled AI provider, so this delegates that power to everyone who can create CoderAIGatewayKey objects here. When false (the default), the controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. | ### CertSecretSelector diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md index 5d76621c5..b24c9c189 100644 --- a/docs/tutorials/getting-started.md +++ b/docs/tutorials/getting-started.md @@ -63,15 +63,16 @@ Then open `http://127.0.0.1:3000` in a browser. ## 5. Clean up (optional) -Delete the `CoderAIProvider` objects first, if you created any, and then the control plane, while the operator still runs. The operator then deletes the providers in Coder, cleans up, and removes its finalizers. A provider deleted after its control plane stays in Coder. After that, remove the bundle. The commands set `BASE` again, because you can be in a new shell. +Delete the `CoderAIProvider` and `CoderAIGatewayKey` objects first, if you created any, and then the control plane, while the operator still runs. The operator then deletes the providers and keys in Coder, cleans up, and removes its finalizers. A provider or key deleted after its control plane stays in Coder. A `CoderAIGatewayKey` can take up to 5 minutes after its last key create. After that, remove the bundle. The commands set `BASE` again, because you can be in a new shell. !!! warning - Delete all other `CoderAIProvider`, `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` resources before you delete the bundle. Delete `CoderAIProvider` resources before their control plane. When you delete the bundle, you also delete the `coder.com` CRDs, and Kubernetes then removes all remaining resources of these kinds in the cluster. When you delete the `coder` namespace, you also delete all other objects in it, for example Secrets and PersistentVolumeClaims. + Delete all other `CoderAIGatewayKey`, `CoderAIProvider`, `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` resources before you delete the bundle. Delete `CoderAIProvider` and `CoderAIGatewayKey` resources before their control plane. When you delete the bundle, you also delete the `coder.com` CRDs, and Kubernetes then removes all remaining resources of these kinds in the cluster. When you delete the `coder` namespace, you also delete all other objects in it, for example Secrets and PersistentVolumeClaims. ```bash BASE="https://raw.githubusercontent.com/coder/coder-k8s/main" kubectl delete coderaiproviders --all -n coder --wait --timeout=120s +kubectl delete coderaigatewaykeys --all -n coder --wait --timeout=420s kubectl delete -f "$BASE/config/samples/coder_v1alpha1_codercontrolplane.yaml" --ignore-not-found kubectl wait --for=delete codercontrolplane/codercontrolplane-sample -n coder --timeout=120s kubectl delete -f "$BASE/dist/install.yaml" --ignore-not-found diff --git a/hack/update-manifests.sh b/hack/update-manifests.sh index a04309fc8..841e7a4f5 100755 --- a/hack/update-manifests.sh +++ b/hack/update-manifests.sh @@ -31,7 +31,7 @@ list_manifests() { # CRDs of kinds whose controller is not registered yet. They go to config/crd/dormant/, which no kustomization, # example, doc or script applies, so the kind is never served without its controller. # Move a file back by deleting its entry when the controller ships. -DORMANT_CRDS=(coder.com_coderaigatewaykeys.yaml coder.com_codermcpservers.yaml coder.com_coderworkspacepools.yaml coder.com_coderworkspacepoolmembers.yaml coder.com_coderworkspaceclaims.yaml) +DORMANT_CRDS=(coder.com_codermcpservers.yaml coder.com_coderworkspacepools.yaml coder.com_coderworkspacepoolmembers.yaml coder.com_coderworkspaceclaims.yaml) # RBAC manifests that only the aggregated API server needs; the controller-only install bundle leaves them out. BUNDLE_EXCLUDED_RBAC=(apiservice-cabundle-role.yaml auth-delegator-binding.yaml authentication-reader-binding.yaml) diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index 360be7674..d0ca9b118 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -134,6 +134,16 @@ func SetupControllers(mgr manager.Manager) error { return fmt.Errorf("unable to create AI provider controller: %w", err) } + aiGatewayKeyReconciler := &controller.CoderAIGatewayKeyReconciler{ + Client: client, + APIReader: mgr.GetAPIReader(), + Scheme: managerScheme, + Clock: clock.RealClock{}, + } + if err := aiGatewayKeyReconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create AI Gateway key controller: %w", err) + } + return nil } diff --git a/internal/controller/coderaigatewaykey_controller.go b/internal/controller/coderaigatewaykey_controller.go index 2c91578e1..8c5b12e73 100644 --- a/internal/controller/coderaigatewaykey_controller.go +++ b/internal/controller/coderaigatewaykey_controller.go @@ -69,10 +69,13 @@ const ( aiGatewayKeyNameAnnotation = "coder.com/ai-gateway-key-name" ) +// +kubebuilder:rbac:groups=coder.com,resources=coderaigatewaykeys,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=coder.com,resources=coderaigatewaykeys/status,verbs=get;update +// +kubebuilder:rbac:groups=coder.com,resources=coderaigatewaykeys/finalizers,verbs=update + // CoderAIGatewayKeyReconciler reconciles CoderAIGatewayKey objects (#156). It -// is not registered yet. This version runs the checks of the observe step -// (plan 4.2) and claims the output Secret. It creates and deletes no Coder -// key. +// issues a Coder AI Gateway key into an owned Secret, rotates it, retires +// replaced keys, and deletes every key of the object when the object goes. type CoderAIGatewayKeyReconciler struct { client.Client // APIReader reads the output Secret and the objects of the reserved-name @@ -105,10 +108,9 @@ func aiGatewayKeyStalled(reason, format string, args ...any) *aiGatewayKeyStep { return &aiGatewayKeyStep{stalled: true, reason: reason, message: fmt.Sprintf(format, args...), requeue: aiGatewayKeyResync} } -// SetupWithManager wires the reconciler into controller-runtime. Nothing -// calls it until activation (K10). A change of the output Secret, or of any -// control plane in the namespace (any of them can reserve the Secret name), -// requeues the object at once. +// SetupWithManager wires the reconciler into controller-runtime. A change of +// the output Secret, or of any control plane in the namespace (any of them +// can reserve the Secret name), requeues the object at once. func (r *CoderAIGatewayKeyReconciler) SetupWithManager(mgr ctrl.Manager) error { if mgr == nil || r.Client == nil || r.APIReader == nil || r.Scheme == nil || r.Clock == nil { return errors.New("assertion failed: AI Gateway key reconciler needs a manager, a client, an API reader, a scheme, and a clock") diff --git a/internal/controller/coderaigatewaykey_controller_test.go b/internal/controller/coderaigatewaykey_controller_test.go index 10d66a4ee..2ec30dd04 100644 --- a/internal/controller/coderaigatewaykey_controller_test.go +++ b/internal/controller/coderaigatewaykey_controller_test.go @@ -374,6 +374,46 @@ func requireAIGatewayKeyReady(t *testing.T, key *coderv1alpha1.CoderAIGatewayKey } } +// The allowKeys gate on a live object (decision D2): without the block or +// with false, the controller creates no key (NotAllowed, zero Coder calls). +// True issues a key. False again stops new keys and rotations, and the +// current key stays in the Secret and in Coder. +func TestCoderAIGatewayKeyAllowKeysGate(t *testing.T) { + t.Parallel() + ctx := context.Background() + env := newAIGatewayKeyEnv(ctx, t, nil) + got, _ := env.reconcile(ctx, t, env.createKey(ctx, t, "coder", testAIGatewaySecret)) + requireAIGatewayKeyStep(t, got, true, "NotAllowed") + setAllowKeys := func(allow bool) { + cp := &coderv1alpha1.CoderControlPlane{} + require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Namespace: env.namespace, Name: "coder"}, cp)) + cp.Spec.AIGateway = &coderv1alpha1.AIGatewaySpec{AllowKeys: allow} + require.NoError(t, k8sClient.Update(ctx, cp)) + } + setAllowKeys(false) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, true, "NotAllowed") + require.Zero(t, env.fake.totalRequests()) + requireNoOutputSecret(ctx, t, env.namespace, testAIGatewaySecret) + + setAllowKeys(true) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyReady(t, got) + current := got.Status.CurrentKey + + setAllowKeys(false) + got.Spec.Rotation.Trigger = "rotate-1" + require.NoError(t, k8sClient.Update(ctx, got)) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, true, "NotAllowed") + require.Equal(t, current.ID, got.Status.CurrentKey.ID, "no rotation") + require.Equal(t, 1, env.fake.requestCount(routeCreateAIGatewayKey)) + require.Len(t, env.fake.aiGatewayKeys(), 1, "the current key stays in Coder") + secret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Namespace: env.namespace, Name: testAIGatewaySecret}, secret)) + require.Equal(t, env.fake.aiGatewayKeys()[0].plaintext, string(secret.Data[coderv1alpha1.CoderAIGatewayKeySecretDataKey]), "the Secret keeps the key") +} + // The object that claimed a shared Secret keeps it. An object created later // in the same second with a lower name ranks older, but it must not stall // the claimant: it reports SecretNotOwned, and the claimant goes on. The diff --git a/internal/controller/codercontrolplane_aigateway_api_test.go b/internal/controller/codercontrolplane_aigateway_api_test.go index 82a09c740..2fb223acf 100644 --- a/internal/controller/codercontrolplane_aigateway_api_test.go +++ b/internal/controller/codercontrolplane_aigateway_api_test.go @@ -47,8 +47,8 @@ func TestControlPlaneAIGatewayAllowKeysRoundTrip(t *testing.T) { } } -// The field has no effect until the CoderAIGatewayKey controller ships: -// turning it on must not roll the Coder Deployment. +// The field gates only the CoderAIGatewayKey controller: turning it on must +// not roll the Coder Deployment. func TestControlPlaneAIGatewayAllowKeysLeavesDeploymentUnchanged(t *testing.T) { ensureGatewaySchemeRegistered(t) ctx := context.Background() diff --git a/internal/controller/controllerapp_setup_test.go b/internal/controller/controllerapp_setup_test.go new file mode 100644 index 000000000..6777d86c8 --- /dev/null +++ b/internal/controller/controllerapp_setup_test.go @@ -0,0 +1,41 @@ +package controller_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/config" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/coder/coder-k8s/internal/app/controllerapp" +) + +// A manager with every controller of the controller app registered starts +// against the envtest API server: no field index is registered twice, every +// watched kind is served, and the caches sync. +func TestSetupControllersStartsEveryController(t *testing.T) { + t.Parallel() + scheme := controllerapp.NewScheme() + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + // Other tests build managers with the same controller names, and + // controller-runtime checks names across the process. + Controller: config.Controller{SkipNameValidation: ptrTo(true)}, + }) + require.NoError(t, err) + require.NoError(t, controllerapp.SetupControllers(mgr)) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- mgr.Start(ctx) }() + syncCtx, syncCancel := context.WithTimeout(ctx, 30*time.Second) + defer syncCancel() + require.True(t, mgr.GetCache().WaitForCacheSync(syncCtx), "the caches sync") + cancel() + require.NoError(t, <-done) +} diff --git a/mkdocs.yml b/mkdocs.yml index 8e80530ae..62e3d4bd8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - Start, stop, and read logs of workspaces: how-to/start-stop-workspaces.md - Test a template version: how-to/test-templates.md - Manage AI providers: how-to/manage-ai-providers.md + - Manage AI Gateway keys: how-to/manage-ai-gateway-keys.md - Gate template promotion with GitOps: how-to/gitops.md - Run the MCP server: how-to/mcp-server.md - Troubleshooting: how-to/troubleshooting.md @@ -51,6 +52,7 @@ nav: - Aggregated API behavior: reference/aggregated-api-behavior.md - API: # BEGIN GENERATED API NAV + - CoderAIGatewayKey: reference/api/coderaigatewaykey.md - CoderAIProvider: reference/api/coderaiprovider.md - CoderControlPlane: reference/api/codercontrolplane.md - CoderProvisioner: reference/api/coderprovisioner.md From 4641a82031aa88329a09eb7bbcc69411c6a25518 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 07:51:16 +0000 Subject: [PATCH 3/5] docs: allow the AI Gateway chart and reloader words Change-Id: I32890aa15aec7a173c3da1edcb0731302f8b2cde Signed-off-by: Thomas Kosiewski --- .cspell.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cspell.json b/.cspell.json index 7473cac53..df441da00 100644 --- a/.cspell.json +++ b/.cspell.json @@ -47,6 +47,8 @@ "codertemplate", "codertemplates", "aibridge", + "aigateway", + "stakater", "coderaigatewaykey", "coderaigatewaykeys", "coderaiprovider", From 812167e302b236e6f7aec032edf20e2d35692d3c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 09:30:02 +0000 Subject: [PATCH 4/5] docs: delete AI Gateway keys before their namespace Change-Id: Idfe28ab7654a28103d15ae2e1a0faa806238c98f Signed-off-by: Thomas Kosiewski --- docs/how-to/manage-ai-gateway-keys.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/how-to/manage-ai-gateway-keys.md b/docs/how-to/manage-ai-gateway-keys.md index 8f2e1c378..80a03755e 100644 --- a/docs/how-to/manage-ai-gateway-keys.md +++ b/docs/how-to/manage-ai-gateway-keys.md @@ -139,9 +139,18 @@ The finalizer stays while Coder cannot confirm the deletion: When the control plane is gone, nothing can delete the keys. The finalizer goes at once. The operator logs `Releasing the finalizer without deleting the keys in Coder` with reason `ControlPlaneGone` and the key IDs: delete those keys with `coder ai-gateway keys delete`. -While the control plane is being deleted, as in a namespace deletion, the operator runs one pass of at most 10 seconds without the settle window, then removes the finalizer whatever the answer. A namespace deletion therefore finishes. A key create that was still in flight at that moment can commit later, and its key stays in Coder. When the pass fails, the log lists the IDs that it did not delete. +While the control plane is being deleted and its operator access still works, the operator runs one pass of at most 10 seconds without the settle window, then removes the finalizer whatever the answer. A key create that was still in flight at that moment can commit later, and its key stays in Coder. When the pass fails, the log lists the IDs that it did not delete. -Delete `CoderAIGatewayKey` objects before their control plane. +!!! warning "Delete the keys before the namespace" + When you delete the namespace, Kubernetes deletes the control plane and the `CoderAIGatewayKey` objects at the same time. The control plane is usually gone before the operator handles the objects, so the finalizers go without a delete, and the keys stay valid in Coder. The namespace deletion finishes, and the operator logs the key IDs with reason `ControlPlaneGone`. + +Delete the `CoderAIGatewayKey` objects first and wait until they are gone, then delete the control plane or the namespace: + +```bash +kubectl -n coder delete coderaigatewaykeys --all --wait --timeout=420s +``` + +If the namespace is already gone, delete the logged keys with `coder ai-gateway keys delete`. ### Stuck finalizer @@ -170,7 +179,7 @@ A deletion that keeps failing never ends by itself. Leave it in one of these way ## Known limits -- **Leftover keys.** A key that commits more than 5 minutes after a lost create answer, or after the object is gone, stays in Coder. Find keys of deleted objects with `coder ai-gateway keys list`: their names start with `ck8s-` and a UID that no `CoderAIGatewayKey` has. +- **Leftover keys.** A key that commits more than 5 minutes after a lost create answer, or after the object is gone, stays in Coder. So do the keys of objects that go together with their control plane, for example in a namespace deletion. Find keys of deleted objects with `coder ai-gateway keys list`: their names start with `ck8s-` and a UID that no `CoderAIGatewayKey` has. - **A short key list.** The operator reads all keys with one list call. When a list answer names the current key but leaves out a replaced key, the operator treats the replaced key as deleted. If the key appears again, the operator deletes it as an orphan, without the rest of its grace period. Coder lists all keys in one answer, so only a proxy between the operator and Coder that changes the answer can cause this. ## Grant access From 1a211c072bf098fe3472da92fe3a707348e61f4e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 09:42:23 +0000 Subject: [PATCH 5/5] fix: describe the AI Gateway key deletion delay and isolate the manager test Change-Id: If7ef13f439aac8f7de5c432cf7ba1f8f45613d38 Signed-off-by: Thomas Kosiewski --- api/v1alpha1/coderaigatewaykey_types.go | 8 ++++++-- config/crd/bases/coder.com_coderaigatewaykeys.yaml | 8 ++++++-- dist/install.yaml | 8 ++++++-- internal/controller/controllerapp_setup_test.go | 11 ++++++++--- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/coderaigatewaykey_types.go b/api/v1alpha1/coderaigatewaykey_types.go index 1f3051ca4..4eea40904 100644 --- a/api/v1alpha1/coderaigatewaykey_types.go +++ b/api/v1alpha1/coderaigatewaykey_types.go @@ -234,8 +234,12 @@ type CoderAIGatewayKeyStatus struct { // CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway // replicas and stores it in a Secret that this object owns. A gateway key // lets its holder load the API key of every enabled AI provider, so write -// access to this kind is admin-only. Deleting the object revokes every key -// that it created at once, and gateways that use them stop within a minute. +// access to this kind is admin-only. Deleting the object deletes every key +// that it created in Coder: after the 5-minute settle window of its last key +// create, and only while the control plane can reach Coder. A failed +// deletion keeps the object and retries. When the control plane is gone, the +// object goes and its keys stay valid in Coder. A gateway stops within a +// minute after its key is deleted. type CoderAIGatewayKey struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/config/crd/bases/coder.com_coderaigatewaykeys.yaml b/config/crd/bases/coder.com_coderaigatewaykeys.yaml index a7dca985b..bd0d3c41b 100644 --- a/config/crd/bases/coder.com_coderaigatewaykeys.yaml +++ b/config/crd/bases/coder.com_coderaigatewaykeys.yaml @@ -37,8 +37,12 @@ spec: CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway replicas and stores it in a Secret that this object owns. A gateway key lets its holder load the API key of every enabled AI provider, so write - access to this kind is admin-only. Deleting the object revokes every key - that it created at once, and gateways that use them stop within a minute. + access to this kind is admin-only. Deleting the object deletes every key + that it created in Coder: after the 5-minute settle window of its last key + create, and only while the control plane can reach Coder. A failed + deletion keeps the object and retries. When the control plane is gone, the + object goes and its keys stay valid in Coder. A gateway stops within a + minute after its key is deleted. properties: apiVersion: description: |- diff --git a/dist/install.yaml b/dist/install.yaml index 9a7637b0d..8138d4996 100644 --- a/dist/install.yaml +++ b/dist/install.yaml @@ -41,8 +41,12 @@ spec: CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway replicas and stores it in a Secret that this object owns. A gateway key lets its holder load the API key of every enabled AI provider, so write - access to this kind is admin-only. Deleting the object revokes every key - that it created at once, and gateways that use them stop within a minute. + access to this kind is admin-only. Deleting the object deletes every key + that it created in Coder: after the 5-minute settle window of its last key + create, and only while the control plane can reach Coder. A failed + deletion keeps the object and retries. When the control plane is gone, the + object goes and its keys stay valid in Coder. A gateway stops within a + minute after its key is deleted. properties: apiVersion: description: |- diff --git a/internal/controller/controllerapp_setup_test.go b/internal/controller/controllerapp_setup_test.go index 6777d86c8..f9144ba7c 100644 --- a/internal/controller/controllerapp_setup_test.go +++ b/internal/controller/controllerapp_setup_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/config" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -15,12 +16,16 @@ import ( // A manager with every controller of the controller app registered starts // against the envtest API server: no field index is registered twice, every -// watched kind is served, and the caches sync. +// watched kind is served, and the caches sync. The test runs serially, and +// its cache sees only an empty namespace, so its reconcilers never touch +// the objects of other tests. func TestSetupControllersStartsEveryController(t *testing.T) { - t.Parallel() + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "setup") scheme := controllerapp.NewScheme() mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}}, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", // Other tests build managers with the same controller names, and @@ -30,7 +35,7 @@ func TestSetupControllersStartsEveryController(t *testing.T) { require.NoError(t, err) require.NoError(t, controllerapp.SetupControllers(mgr)) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- mgr.Start(ctx) }() syncCtx, syncCancel := context.WithTimeout(ctx, 30*time.Second)