From 0ab316eafee09c4c7670327b1d7c2848f35d0aee Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 03:36:17 +0000 Subject: [PATCH 01/10] =?UTF-8?q?=F0=9F=A4=96=20feat:=20delete=20the=20Cod?= =?UTF-8?q?er=20provider=20of=20a=20deleted=20CoderAIProvider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The finalizer now deletes the Coder provider that the object proved it created: by the recorded ID, or after a read that finds this object's nonce when only a create marker exists. 204 or 404 releases the finalizer, and a failed delete retries every 60 seconds. When the control plane is gone, the finalizer goes at once. While the control plane is being deleted, one bounded delete attempt runs first (plan D12). Refs #154 Change-Id: I6e99fcb5d88121cc46f132599a13b9f7f82d2768 Signed-off-by: Thomas Kosiewski --- .../controller/coderaiprovider_controller.go | 15 +- .../coderaiprovider_controller_test.go | 19 +- internal/controller/coderaiprovider_delete.go | 102 ++++++++ .../controller/coderaiprovider_delete_test.go | 225 ++++++++++++++++++ internal/controller/controlplane_access.go | 9 +- 5 files changed, 353 insertions(+), 17 deletions(-) create mode 100644 internal/controller/coderaiprovider_delete.go create mode 100644 internal/controller/coderaiprovider_delete_test.go diff --git a/internal/controller/coderaiprovider_controller.go b/internal/controller/coderaiprovider_controller.go index 1afbcb10d..b23d8c724 100644 --- a/internal/controller/coderaiprovider_controller.go +++ b/internal/controller/coderaiprovider_controller.go @@ -152,15 +152,16 @@ func (r *CoderAIProviderReconciler) Reconcile(ctx context.Context, req ctrl.Requ if !controllerutil.ContainsFinalizer(provider, coderv1alpha1.CoderAIProviderCleanupFinalizer) { return ctrl.Result{}, nil } - if provider.Status.ProviderID == "" && provider.Status.CreateAttemptTime == nil { - // Nothing of this provider can exist in Coder. + release, step, err := r.deleteStep(ctx, provider) + if err != nil { + return ctrl.Result{}, err + } + if release { + r.sentKeys.Delete(provider.UID) return ctrl.Result{}, r.patchFinalizer(ctx, provider, false) } - // Keep the finalizer: this controller version cannot prove what Coder - // holds, so it neither deletes nor forgets the provider. - applyAIProviderStep(provider, aiProviderWait(0, "DeletePending", - "Provider %s can exist in Coder. This controller version keeps the finalizer and deletes nothing.", provider.Name)) - return ctrl.Result{}, r.writeStatus(ctx, provider, before) + applyAIProviderStep(provider, step) + return ctrl.Result{RequeueAfter: step.requeue}, r.writeStatus(ctx, provider, before) } if err := r.patchFinalizer(ctx, provider, true); err != nil { diff --git a/internal/controller/coderaiprovider_controller_test.go b/internal/controller/coderaiprovider_controller_test.go index 562ebd96e..515046e0a 100644 --- a/internal/controller/coderaiprovider_controller_test.go +++ b/internal/controller/coderaiprovider_controller_test.go @@ -333,15 +333,16 @@ func TestCoderAIProviderFinalizer(t *testing.T) { require.NoError(t, err) require.True(t, apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(got), got))) - // A provider with a Coder ID keeps its finalizer: this version deletes nothing. - kept := env.createProvider(ctx, t, nil) - kept, _ = env.reconcile(ctx, t, kept) - kept.Status.ProviderID = "5d0c6c2e-3a4d-4c55-9d2f-3f1b1f0c9a11" - require.NoError(t, k8sClient.Status().Update(ctx, kept)) - require.NoError(t, k8sClient.Delete(ctx, kept)) - kept, _ = env.reconcile(ctx, t, kept) - require.Equal(t, []string{coderv1alpha1.CoderAIProviderCleanupFinalizer}, kept.Finalizers) - requireAIProviderStep(t, kept, false, "DeletePending") + // A provider with a Coder ID whose control plane does not exist: nothing + // can delete it, so the finalizer goes without a Coder request. + gone := env.createProvider(ctx, t, nil) + gone, _ = env.reconcile(ctx, t, gone) + gone.Status.ProviderID = "5d0c6c2e-3a4d-4c55-9d2f-3f1b1f0c9a11" + require.NoError(t, k8sClient.Status().Update(ctx, gone)) + require.NoError(t, k8sClient.Delete(ctx, gone)) + _, err = env.r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gone)}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(gone), gone))) require.Zero(t, env.fake.totalRequests()) } diff --git a/internal/controller/coderaiprovider_delete.go b/internal/controller/coderaiprovider_delete.go new file mode 100644 index 000000000..2475f3f4f --- /dev/null +++ b/internal/controller/coderaiprovider_delete.go @@ -0,0 +1,102 @@ +package controller + +import ( + "context" + "time" + + "github.com/coder/coder/v2/codersdk" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" +) + +const ( + // aiProviderDeletePoll is the retry of a delete that Coder did not + // confirm. A delete has no deadline (plan 4.6). + aiProviderDeletePoll = 60 * time.Second + // aiProviderBoundedDeleteTimeout bounds the one delete attempt while the + // control plane is being deleted (plan D12). + aiProviderBoundedDeleteTimeout = 10 * time.Second +) + +// deleteStep handles an object that is being deleted. It deletes in Coder +// only a provider that this object proved it created (plan S7), and it +// returns release when the finalizer can go. Otherwise the step says why +// the finalizer stays. +func (r *CoderAIProviderReconciler) deleteStep(ctx context.Context, provider *coderv1alpha1.CoderAIProvider) (release bool, step *aiProviderStep, err error) { + if provider.Status.ProviderID == "" && provider.Status.CreateAttemptTime == nil { + // Nothing of this object can exist in Coder. + return true, nil, nil + } + ref := types.NamespacedName{Namespace: provider.Namespace, Name: provider.Spec.ControlPlaneRef.Name} + controlPlane := &coderv1alpha1.CoderControlPlane{} + if err := r.Get(ctx, ref, controlPlane); apierrors.IsNotFound(err) { + return r.releaseWithoutDelete(ctx, provider, "CoderControlPlane is gone"), nil, nil + } else if err != nil { + return false, nil, err + } + deleting := !controlPlane.DeletionTimestamp.IsZero() + sdk, _, wait, err := controlPlaneCoderClientFor(ctx, r.Client, ref.Namespace, ref.Name, deleting) + switch { + case err != nil: + return false, nil, err + case wait != nil && deleting: + return r.releaseWithoutDelete(ctx, provider, "CoderControlPlane is being deleted without operator access"), nil, nil + case wait != nil: + return false, aiProviderWait(aiProviderDeletePoll, wait.reason, "%s The controller deletes provider %s once it can call Coder.", wait.message, provider.Name), nil + } + if deleting { + // One bounded attempt, then release, so the namespace deletion + // finishes (plan D12). + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, aiProviderBoundedDeleteTimeout) + defer cancel() + } + step = r.deleteInCoder(ctx, sdk, provider) + if step != nil && deleting { + return r.releaseWithoutDelete(ctx, provider, "the one delete attempt while CoderControlPlane is being deleted failed: "+step.message), nil, nil + } + return step == nil, step, nil +} + +// deleteInCoder deletes the provider of this object in Coder and returns nil +// when Coder holds none. With only a create marker, a read decides first: +// only the nonce and the type prove that the provider is this object's. +func (r *CoderAIProviderReconciler) deleteInCoder(ctx context.Context, sdk *codersdk.Client, provider *coderv1alpha1.CoderAIProvider) *aiProviderStep { + id := provider.Status.ProviderID + if id == "" { + found, err := sdk.AIProvider(ctx, provider.Name) + switch { + case isCoderNotFound(err): + // A 404 proves absence only for a site owner. + return requireSiteOwner(ctx, sdk) + case err != nil: + return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "get provider %s: %s", provider.Name, coderErrorSummary(err)) + } + if err := coderAnswerFor("AI provider", provider.Name, found.Name); err != nil { + log.FromContext(ctx).Error(err, "Coder answered something other than the controller asked for") + return aiProviderWait(aiProviderDeletePoll, "CoderAnswerMismatch", "%s", err.Error()) + } + if found.DisplayName != aiProviderNonce(provider) || string(found.Type) != provider.Spec.Type { + // Someone else's provider holds the name, so no create of this + // object committed. The controller never deletes it. + return nil + } + id = found.ID.String() + } + if err := sdk.DeleteAIProvider(ctx, id); err != nil && !isCoderNotFound(err) { + return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "delete provider %s (ID %s): %s", provider.Name, id, coderErrorSummary(err)) + } + return nil +} + +// releaseWithoutDelete logs what can stay in Coder, because nothing can +// delete it any more, and returns true (ControlPlaneGone). +func (r *CoderAIProviderReconciler) releaseWithoutDelete(ctx context.Context, provider *coderv1alpha1.CoderAIProvider, why string) bool { + log.FromContext(ctx).Info("Releasing the finalizer without a delete in Coder: the provider can stay in the Coder database", + "reason", "ControlPlaneGone", "why", why, "provider", provider.Name, "providerID", provider.Status.ProviderID, + "createPending", provider.Status.CreateAttemptTime != nil) + return true +} diff --git a/internal/controller/coderaiprovider_delete_test.go b/internal/controller/coderaiprovider_delete_test.go new file mode 100644 index 000000000..d79c56bba --- /dev/null +++ b/internal/controller/coderaiprovider_delete_test.go @@ -0,0 +1,225 @@ +package controller_test + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/coder/coder/v2/codersdk" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" +) + +// deleteAIProvider deletes the object and runs one reconcile. It returns +// the object, or nil when the finalizer went and the object is gone. +func (env aiProviderEnv) deleteAIProvider(ctx context.Context, t *testing.T, provider *coderv1alpha1.CoderAIProvider) (*coderv1alpha1.CoderAIProvider, ctrl.Result) { + t.Helper() + require.NoError(t, client.IgnoreNotFound(k8sClient.Delete(ctx, provider))) + return env.reconcileDeleting(ctx, t, provider) +} + +func (env aiProviderEnv) reconcileDeleting(ctx context.Context, t *testing.T, provider *coderv1alpha1.CoderAIProvider) (*coderv1alpha1.CoderAIProvider, ctrl.Result) { + t.Helper() + result, err := env.r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(provider)}) + require.NoError(t, err) + got := &coderv1alpha1.CoderAIProvider{} + err = k8sClient.Get(ctx, client.ObjectKeyFromObject(provider), got) + if apierrors.IsNotFound(err) { + return nil, result + } + require.NoError(t, err) + return got, result +} + +// holdControlPlane puts a finalizer on the control plane and deletes it, so +// it stays Terminating like a control plane whose own cleanup still runs. +func holdControlPlane(ctx context.Context, t *testing.T, namespace string) { + t.Helper() + controlPlane := &coderv1alpha1.CoderControlPlane{} + key := client.ObjectKey{Namespace: namespace, Name: "coder"} + require.NoError(t, k8sClient.Get(ctx, key, controlPlane)) + controlPlane.Finalizers = append(controlPlane.Finalizers, "example.com/hold") + require.NoError(t, k8sClient.Update(ctx, controlPlane)) + require.NoError(t, k8sClient.Delete(ctx, controlPlane)) + t.Cleanup(func() { + if k8sClient.Get(context.Background(), key, controlPlane) == nil { + controlPlane.Finalizers = nil + _ = k8sClient.Update(context.Background(), controlPlane) + } + }) +} + +// Plan S7 and 4.6: the controller deletes by the recorded ID, and 204 or 404 +// releases the finalizer. A failed delete keeps the finalizer and retries +// every 60 seconds with no deadline. +func TestCoderAIProviderDelete(t *testing.T) { + t.Parallel() + t.Run("deleted", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Equal(t, 1, env.fake.requestCount(routeDeleteAIProvider)) + _, found := env.fake.aiProvider("openai") + require.False(t, found) + }) + t.Run("already gone in Coder", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + require.NoError(t, env.fake.client(t, 2*time.Second).DeleteAIProvider(ctx, provider.Status.ProviderID)) + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "a 404 releases the finalizer") + require.Equal(t, 2, env.fake.requestCount(routeDeleteAIProvider)) + }) + t.Run("Coder unreachable", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + env.fake.failNext(routeDeleteAIProvider, fakeFault{Status: http.StatusBadGateway}) + got, result := env.deleteAIProvider(ctx, t, provider) + require.NotNil(t, got) + require.Equal(t, []string{coderv1alpha1.CoderAIProviderCleanupFinalizer}, got.Finalizers) + require.Contains(t, requireAIProviderStep(t, got, false, "CoderUnavailable"), "delete provider openai") + require.Equal(t, 60*time.Second, result.RequeueAfter) + _, found := env.fake.aiProvider("openai") + require.True(t, found) + + got, _ = env.reconcileDeleting(ctx, t, got) + require.Nil(t, got) + require.Equal(t, 2, env.fake.requestCount(routeDeleteAIProvider)) + }) + t.Run("control plane not ready", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + controlPlane := &coderv1alpha1.CoderControlPlane{} + require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Namespace: env.namespace, Name: "coder"}, controlPlane)) + controlPlane.Status.OperatorAccessReady = false + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + requests := env.fake.totalRequests() + got, result := env.deleteAIProvider(ctx, t, provider) + require.NotNil(t, got) + requireAIProviderStep(t, got, false, "OperatorAccessNotReady") + require.Equal(t, 60*time.Second, result.RequeueAfter) + require.Equal(t, requests, env.fake.totalRequests(), "no Coder request") + }) +} + +// Plan S7: with only a create marker, a read decides. The controller deletes +// a provider that carries this object's nonce, and never one that does not. +func TestCoderAIProviderDeletePendingCreate(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + displayName string // of a provider that holds the name in Coder; empty for none + deletes int + }{ + {name: "create committed", displayName: "nonce", deletes: 1}, + {name: "nothing created", deletes: 0}, + {name: "someone else's provider", displayName: "Team OpenAI", deletes: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newAIProviderCreateEnv(ctx, t) + if tc.displayName != "" { + displayName := tc.displayName + if displayName == "nonce" { + displayName = "coder-k8s:" + string(provider.UID) + } + _, err := env.fake.client(t, 2*time.Second).CreateAIProvider(ctx, codersdk.CreateAIProviderRequest{ + Type: codersdk.AIProviderTypeOpenAI, Name: provider.Name, DisplayName: displayName, + BaseURL: provider.Spec.BaseURL, APIKeys: []string{testAIProviderKey}, + }) + require.NoError(t, err) + } + provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} + require.NoError(t, k8sClient.Update(ctx, provider)) + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} + require.NoError(t, k8sClient.Status().Update(ctx, provider)) + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Equal(t, tc.deletes, env.fake.requestCount(routeDeleteAIProvider)) + _, found := env.fake.aiProvider(provider.Name) + require.Equal(t, tc.displayName != "" && tc.deletes == 0, found, "only this object's provider is deleted") + }) + } +} + +// A 404 proves that no create of this object committed only when the +// operator user is a site owner. Otherwise the finalizer stays. +func TestCoderAIProviderDeletePendingCreateNeedsSiteOwner(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newAIProviderCreateEnv(ctx, t) + provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} + require.NoError(t, k8sClient.Update(ctx, provider)) + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} + require.NoError(t, k8sClient.Status().Update(ctx, provider)) + env.fake.updateUser(env.fake.operatorID, func(u *codersdk.User) { u.Roles = []codersdk.SlimRole{} }) + got, _ := env.deleteAIProvider(ctx, t, provider) + require.NotNil(t, got) + require.Contains(t, requireAIProviderStep(t, got, false, "ConfirmingCreate"), "not a site owner") + require.Equal(t, []string{coderv1alpha1.CoderAIProviderCleanupFinalizer}, got.Finalizers) + require.Zero(t, env.fake.requestCount(routeDeleteAIProvider)) +} + +// Plan D12 and S7: when the control plane is gone, nothing can delete the +// provider, so the finalizer goes at once with zero Coder requests. While the +// control plane is being deleted with working operator access, the +// controller sends exactly one bounded delete, and then releases the +// finalizer whatever the answer. A namespace deletion therefore finishes. +func TestCoderAIProviderDeleteWithControlPlaneGoing(t *testing.T) { + t.Parallel() + t.Run("control plane gone", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + // The namespace controller deletes every object in the namespace. + require.NoError(t, k8sClient.Delete(ctx, &coderv1alpha1.CoderControlPlane{ObjectMeta: metav1.ObjectMeta{Namespace: env.namespace, Name: "coder"}})) + requests := env.fake.totalRequests() + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Equal(t, requests, env.fake.totalRequests(), "no Coder request") + _, found := env.fake.aiProvider("openai") + require.True(t, found, "the provider stays in Coder") + }) + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "control plane being deleted", true: "control plane being deleted, delete fails"}[fail], func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + holdControlPlane(ctx, t, env.namespace) + if fail { + env.fake.failNext(routeDeleteAIProvider, fakeFault{Status: http.StatusBadGateway}) + } + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Equal(t, 1, env.fake.requestCount(routeDeleteAIProvider), "exactly one delete attempt") + _, found := env.fake.aiProvider("openai") + require.Equal(t, fail, found) + }) + } + t.Run("control plane being deleted without operator access", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + controlPlane := &coderv1alpha1.CoderControlPlane{} + require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Namespace: env.namespace, Name: "coder"}, controlPlane)) + controlPlane.Status.OperatorAccessReady = false + require.NoError(t, k8sClient.Status().Update(ctx, controlPlane)) + holdControlPlane(ctx, t, env.namespace) + requests := env.fake.totalRequests() + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Equal(t, requests, env.fake.totalRequests(), "no Coder request") + }) +} diff --git a/internal/controller/controlplane_access.go b/internal/controller/controlplane_access.go index 6558dd9d2..74e5797c3 100644 --- a/internal/controller/controlplane_access.go +++ b/internal/controller/controlplane_access.go @@ -38,6 +38,13 @@ func newControlPlaneWait(reason, format string, args ...any) *controlPlaneWait { // its operator token, and returns a Coder client with that token. When the // control plane cannot serve Coder calls yet, it returns a wait and no client. func controlPlaneCoderClient(ctx context.Context, reader client.Reader, namespace, name string) (*codersdk.Client, *coderv1alpha1.CoderControlPlane, *controlPlaneWait, error) { + return controlPlaneCoderClientFor(ctx, reader, namespace, name, false) +} + +// controlPlaneCoderClientFor is controlPlaneCoderClient. With allowDeleting +// it also returns a client for a control plane that is being deleted, while +// its operator access still works. Only cleanups use that (plan D12). +func controlPlaneCoderClientFor(ctx context.Context, reader client.Reader, namespace, name string, allowDeleting bool) (*codersdk.Client, *coderv1alpha1.CoderControlPlane, *controlPlaneWait, error) { if reader == nil { return nil, nil, nil, errors.New("assertion failed: control plane access needs a Kubernetes reader") } @@ -52,7 +59,7 @@ func controlPlaneCoderClient(ctx context.Context, reader client.Reader, namespac return nil, nil, nil, fmt.Errorf("get codercontrolplane %s: %w", key, err) } switch { - case !controlPlane.DeletionTimestamp.IsZero(): + case !controlPlane.DeletionTimestamp.IsZero() && !allowDeleting: return nil, nil, newControlPlaneWait("ControlPlaneNotReady", "CoderControlPlane %s is being deleted.", key.Name), nil case controlPlane.Status.URL == "": return nil, nil, newControlPlaneWait("ControlPlaneNotReady", "CoderControlPlane %s has no status.url yet.", key.Name), nil From 0b7e74e1503eb07422380643b7b55ead23191f3a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:07:44 +0000 Subject: [PATCH 02/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20bound=20the=20Coder?= =?UTF-8?q?AIProvider=20delete=20waits,=20and=20reject=20an=20ID-less=20lo?= =?UTF-8?q?okup=20answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: Ia687ee9bc423b12876f8827a959c735b556f1c56 Signed-off-by: Thomas Kosiewski --- internal/controller/coderaiprovider_delete.go | 33 ++++++++--- .../controller/coderaiprovider_delete_test.go | 55 ++++++++++++++++++- 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/internal/controller/coderaiprovider_delete.go b/internal/controller/coderaiprovider_delete.go index 2475f3f4f..3c4de5488 100644 --- a/internal/controller/coderaiprovider_delete.go +++ b/internal/controller/coderaiprovider_delete.go @@ -2,9 +2,11 @@ package controller import ( "context" + "fmt" "time" "github.com/coder/coder/v2/codersdk" + "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" @@ -40,6 +42,9 @@ func (r *CoderAIProviderReconciler) deleteStep(ctx context.Context, provider *co deleting := !controlPlane.DeletionTimestamp.IsZero() sdk, _, wait, err := controlPlaneCoderClientFor(ctx, r.Client, ref.Namespace, ref.Name, deleting) switch { + case err != nil && deleting: + // One attempt only, also when the access setup fails (plan D12). + return r.releaseWithoutDelete(ctx, provider, "the Coder access of CoderControlPlane, which is being deleted, failed: "+err.Error()), nil, nil case err != nil: return false, nil, err case wait != nil && deleting: @@ -54,7 +59,9 @@ func (r *CoderAIProviderReconciler) deleteStep(ctx context.Context, provider *co ctx, cancel = context.WithTimeout(ctx, aiProviderBoundedDeleteTimeout) defer cancel() } - step = r.deleteInCoder(ctx, sdk, provider) + if step, err = r.deleteInCoder(ctx, sdk, provider); err != nil { + return false, nil, err + } if step != nil && deleting { return r.releaseWithoutDelete(ctx, provider, "the one delete attempt while CoderControlPlane is being deleted failed: "+step.message), nil, nil } @@ -64,32 +71,40 @@ func (r *CoderAIProviderReconciler) deleteStep(ctx context.Context, provider *co // deleteInCoder deletes the provider of this object in Coder and returns nil // when Coder holds none. With only a create marker, a read decides first: // only the nonce and the type prove that the provider is this object's. -func (r *CoderAIProviderReconciler) deleteInCoder(ctx context.Context, sdk *codersdk.Client, provider *coderv1alpha1.CoderAIProvider) *aiProviderStep { +func (r *CoderAIProviderReconciler) deleteInCoder(ctx context.Context, sdk *codersdk.Client, provider *coderv1alpha1.CoderAIProvider) (*aiProviderStep, error) { id := provider.Status.ProviderID if id == "" { found, err := sdk.AIProvider(ctx, provider.Name) switch { case isCoderNotFound(err): - // A 404 proves absence only for a site owner. - return requireSiteOwner(ctx, sdk) + // A 404 proves absence only for a site owner. A role change in + // Coder has no watch, so the wait uses the delete poll. + if step := requireSiteOwner(ctx, sdk); step != nil { + step.requeue = aiProviderDeletePoll + return step, nil + } + return nil, nil case err != nil: - return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "get provider %s: %s", provider.Name, coderErrorSummary(err)) + return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "get provider %s: %s", provider.Name, coderErrorSummary(err)), nil } if err := coderAnswerFor("AI provider", provider.Name, found.Name); err != nil { log.FromContext(ctx).Error(err, "Coder answered something other than the controller asked for") - return aiProviderWait(aiProviderDeletePoll, "CoderAnswerMismatch", "%s", err.Error()) + return aiProviderWait(aiProviderDeletePoll, "CoderAnswerMismatch", "%s", err.Error()), nil } if found.DisplayName != aiProviderNonce(provider) || string(found.Type) != provider.Spec.Type { // Someone else's provider holds the name, so no create of this // object committed. The controller never deletes it. - return nil + return nil, nil + } + if found.ID == uuid.Nil { + return nil, fmt.Errorf("assertion failed: Coder answered AI provider %s without an ID", provider.Name) } id = found.ID.String() } if err := sdk.DeleteAIProvider(ctx, id); err != nil && !isCoderNotFound(err) { - return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "delete provider %s (ID %s): %s", provider.Name, id, coderErrorSummary(err)) + return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "delete provider %s (ID %s): %s", provider.Name, id, coderErrorSummary(err)), nil } - return nil + return nil, nil } // releaseWithoutDelete logs what can stay in Coder, because nothing can diff --git a/internal/controller/coderaiprovider_delete_test.go b/internal/controller/coderaiprovider_delete_test.go index d79c56bba..0f1f6e895 100644 --- a/internal/controller/coderaiprovider_delete_test.go +++ b/internal/controller/coderaiprovider_delete_test.go @@ -7,11 +7,14 @@ import ( "time" "github.com/coder/coder/v2/codersdk" + "github.com/google/uuid" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" coderv1alpha1 "github.com/coder/coder-k8s/api/v1alpha1" ) @@ -165,13 +168,43 @@ func TestCoderAIProviderDeletePendingCreateNeedsSiteOwner(t *testing.T) { provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} require.NoError(t, k8sClient.Status().Update(ctx, provider)) env.fake.updateUser(env.fake.operatorID, func(u *codersdk.User) { u.Roles = []codersdk.SlimRole{} }) - got, _ := env.deleteAIProvider(ctx, t, provider) + got, result := env.deleteAIProvider(ctx, t, provider) require.NotNil(t, got) require.Contains(t, requireAIProviderStep(t, got, false, "ConfirmingCreate"), "not a site owner") + require.Equal(t, 60*time.Second, result.RequeueAfter, "a delete waits with the delete poll") require.Equal(t, []string{coderv1alpha1.CoderAIProviderCleanupFinalizer}, got.Finalizers) require.Zero(t, env.fake.requestCount(routeDeleteAIProvider)) } +// A name lookup that answers this object's provider without an ID is +// impossible. The reconcile fails fast, deletes nothing and keeps the +// finalizer. +func TestCoderAIProviderDeletePendingCreateAnswerWithoutID(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newAIProviderCreateEnv(ctx, t) + _, err := env.fake.client(t, 2*time.Second).CreateAIProvider(ctx, codersdk.CreateAIProviderRequest{ + Type: codersdk.AIProviderTypeOpenAI, Name: provider.Name, DisplayName: "coder-k8s:" + string(provider.UID), + BaseURL: provider.Spec.BaseURL, APIKeys: []string{testAIProviderKey}, + }) + require.NoError(t, err) + provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} + require.NoError(t, k8sClient.Update(ctx, provider)) + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} + require.NoError(t, k8sClient.Status().Update(ctx, provider)) + env.fake.failNext(routeAIProvider, fakeFault{Rewrite: func(answer any) any { + p := answer.(codersdk.AIProvider) + p.ID = uuid.Nil + return p + }}) + require.NoError(t, k8sClient.Delete(ctx, provider)) + _, err = env.r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(provider)}) + require.ErrorContains(t, err, "assertion failed: Coder answered AI provider openai without an ID") + require.NoError(t, k8sClient.Get(ctx, client.ObjectKeyFromObject(provider), provider)) + require.Equal(t, []string{coderv1alpha1.CoderAIProviderCleanupFinalizer}, provider.Finalizers) + require.Zero(t, env.fake.requestCount(routeDeleteAIProvider)) +} + // Plan D12 and S7: when the control plane is gone, nothing can delete the // provider, so the finalizer goes at once with zero Coder requests. While the // control plane is being deleted with working operator access, the @@ -208,6 +241,26 @@ func TestCoderAIProviderDeleteWithControlPlaneGoing(t *testing.T) { require.Equal(t, fail, found) }) } + t.Run("control plane being deleted, operator token read fails", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newReadyAIProvider(ctx, t) + holdControlPlane(ctx, t, env.namespace) + base, err := client.NewWithWatch(cfg, client.Options{Scheme: scheme}) + require.NoError(t, err) + env.r.Client = interceptor.NewClient(base, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "coder-operator-token" { + return apierrors.NewTimeoutError("the server was unable to return a response in the time allotted", 1) + } + return c.Get(ctx, key, obj, opts...) + }, + }) + requests := env.fake.totalRequests() + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Equal(t, requests, env.fake.totalRequests(), "no Coder request") + }) t.Run("control plane being deleted without operator access", func(t *testing.T) { t.Parallel() ctx := context.Background() From 4388b96edcb704d4f10be194243f41c394dcde5b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:25:37 +0000 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20release=20a=20Coder?= =?UTF-8?q?AIProvider=20after=20an=20impossible=20answer=20while=20its=20c?= =?UTF-8?q?ontrol=20plane=20is=20being=20deleted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I99f643fee0650ef981484202e3d35d8d67f10a7d Signed-off-by: Thomas Kosiewski --- internal/controller/coderaiprovider_delete.go | 7 ++++++- .../controller/coderaiprovider_delete_test.go | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/controller/coderaiprovider_delete.go b/internal/controller/coderaiprovider_delete.go index 3c4de5488..189240937 100644 --- a/internal/controller/coderaiprovider_delete.go +++ b/internal/controller/coderaiprovider_delete.go @@ -59,7 +59,12 @@ func (r *CoderAIProviderReconciler) deleteStep(ctx context.Context, provider *co ctx, cancel = context.WithTimeout(ctx, aiProviderBoundedDeleteTimeout) defer cancel() } - if step, err = r.deleteInCoder(ctx, sdk, provider); err != nil { + if step, err = r.deleteInCoder(ctx, sdk, provider); err != nil && deleting { + // The attempt ended with an impossible answer. Log it, and still + // release: the one attempt is over. + log.FromContext(ctx).Error(err, "The one delete attempt while CoderControlPlane is being deleted got an impossible answer") + return r.releaseWithoutDelete(ctx, provider, "the one delete attempt while CoderControlPlane is being deleted failed: "+err.Error()), nil, nil + } else if err != nil { return false, nil, err } if step != nil && deleting { diff --git a/internal/controller/coderaiprovider_delete_test.go b/internal/controller/coderaiprovider_delete_test.go index 0f1f6e895..d5cce8f05 100644 --- a/internal/controller/coderaiprovider_delete_test.go +++ b/internal/controller/coderaiprovider_delete_test.go @@ -178,9 +178,19 @@ func TestCoderAIProviderDeletePendingCreateNeedsSiteOwner(t *testing.T) { // A name lookup that answers this object's provider without an ID is // impossible. The reconcile fails fast, deletes nothing and keeps the -// finalizer. +// finalizer. While the control plane is being deleted, the one attempt is +// over, so the finalizer goes. func TestCoderAIProviderDeletePendingCreateAnswerWithoutID(t *testing.T) { t.Parallel() + for _, deleting := range []bool{false, true} { + t.Run(map[bool]string{false: "control plane ready", true: "control plane being deleted"}[deleting], func(t *testing.T) { + t.Parallel() + testAIProviderDeleteAnswerWithoutID(t, deleting) + }) + } +} + +func testAIProviderDeleteAnswerWithoutID(t *testing.T, deleting bool) { ctx := context.Background() env, _, provider := newAIProviderCreateEnv(ctx, t) _, err := env.fake.client(t, 2*time.Second).CreateAIProvider(ctx, codersdk.CreateAIProviderRequest{ @@ -197,6 +207,13 @@ func TestCoderAIProviderDeletePendingCreateAnswerWithoutID(t *testing.T) { p.ID = uuid.Nil return p }}) + if deleting { + holdControlPlane(ctx, t, env.namespace) + got, _ := env.deleteAIProvider(ctx, t, provider) + require.Nil(t, got, "the finalizer went") + require.Zero(t, env.fake.requestCount(routeDeleteAIProvider)) + return + } require.NoError(t, k8sClient.Delete(ctx, provider)) _, err = env.r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(provider)}) require.ErrorContains(t, err, "assertion failed: Coder answered AI provider openai without an ID") From f2488c95ce1f1f943b637d3d930f7c45c5e8a053 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:39:16 +0000 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=A4=96=20fix:=20wait=20out=20a=20re?= =?UTF-8?q?cent=20create=20before=20a=20CoderAIProvider=20delete=20trusts?= =?UTF-8?q?=20a=20404?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: Id18d664b10707befd7e2c4bc2af78012f3ef8fb4 Signed-off-by: Thomas Kosiewski --- internal/controller/coderaiprovider_delete.go | 9 +++++ .../controller/coderaiprovider_delete_test.go | 39 +++++++++++++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/internal/controller/coderaiprovider_delete.go b/internal/controller/coderaiprovider_delete.go index 189240937..b1c024f82 100644 --- a/internal/controller/coderaiprovider_delete.go +++ b/internal/controller/coderaiprovider_delete.go @@ -21,6 +21,9 @@ const ( // aiProviderBoundedDeleteTimeout bounds the one delete attempt while the // control plane is being deleted (plan D12). aiProviderBoundedDeleteTimeout = 10 * time.Second + // aiProviderCreateSettle is how long after a create request a 404 does + // not yet prove absence: the request can still be in flight in Coder. + aiProviderCreateSettle = 2 * time.Minute ) // deleteStep handles an object that is being deleted. It deletes in Coder @@ -88,6 +91,12 @@ func (r *CoderAIProviderReconciler) deleteInCoder(ctx context.Context, sdk *code step.requeue = aiProviderDeletePoll return step, nil } + if wait := aiProviderCreateSettle - time.Since(provider.Status.CreateAttemptTime.Time); wait > 0 { + // A create sent moments ago can still commit after this read. + return aiProviderWait(wait, "ConfirmingCreate", + "Provider %s is not in Coder, but its create request was sent at %s and can still commit. The controller reads again before it releases the finalizer.", + provider.Name, provider.Status.CreateAttemptTime.UTC().Format(time.RFC3339)), nil + } return nil, nil case err != nil: return aiProviderWait(aiProviderDeletePoll, "CoderUnavailable", "get provider %s: %s", provider.Name, coderErrorSummary(err)), nil diff --git a/internal/controller/coderaiprovider_delete_test.go b/internal/controller/coderaiprovider_delete_test.go index d5cce8f05..598ad2d0f 100644 --- a/internal/controller/coderaiprovider_delete_test.go +++ b/internal/controller/coderaiprovider_delete_test.go @@ -146,7 +146,7 @@ func TestCoderAIProviderDeletePendingCreate(t *testing.T) { } provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} require.NoError(t, k8sClient.Update(ctx, provider)) - provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now().Add(-3 * time.Minute)} require.NoError(t, k8sClient.Status().Update(ctx, provider)) got, _ := env.deleteAIProvider(ctx, t, provider) require.Nil(t, got, "the finalizer went") @@ -165,7 +165,7 @@ func TestCoderAIProviderDeletePendingCreateNeedsSiteOwner(t *testing.T) { env, _, provider := newAIProviderCreateEnv(ctx, t) provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} require.NoError(t, k8sClient.Update(ctx, provider)) - provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now().Add(-3 * time.Minute)} require.NoError(t, k8sClient.Status().Update(ctx, provider)) env.fake.updateUser(env.fake.operatorID, func(u *codersdk.User) { u.Roles = []codersdk.SlimRole{} }) got, result := env.deleteAIProvider(ctx, t, provider) @@ -176,6 +176,39 @@ func TestCoderAIProviderDeletePendingCreateNeedsSiteOwner(t *testing.T) { require.Zero(t, env.fake.requestCount(routeDeleteAIProvider)) } +// A 404 right after a create request does not prove absence: the request +// can still commit. The finalizer stays until the settle window ends, and a +// create that commits in the meantime is deleted. +func TestCoderAIProviderDeleteRecentCreate(t *testing.T) { + t.Parallel() + ctx := context.Background() + env, _, provider := newAIProviderCreateEnv(ctx, t) + provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} + require.NoError(t, k8sClient.Update(ctx, provider)) + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now().Add(-30 * time.Second)} + require.NoError(t, k8sClient.Status().Update(ctx, provider)) + got, result := env.deleteAIProvider(ctx, t, provider) + require.NotNil(t, got) + require.Contains(t, requireAIProviderStep(t, got, false, "ConfirmingCreate"), "can still commit") + require.Equal(t, []string{coderv1alpha1.CoderAIProviderCleanupFinalizer}, got.Finalizers) + require.Greater(t, result.RequeueAfter, time.Duration(0)) + require.LessOrEqual(t, result.RequeueAfter, 90*time.Second) + + // The create commits late. The next read after the window finds it. + _, err := env.fake.client(t, 2*time.Second).CreateAIProvider(ctx, codersdk.CreateAIProviderRequest{ + Type: codersdk.AIProviderTypeOpenAI, Name: provider.Name, DisplayName: "coder-k8s:" + string(provider.UID), + BaseURL: provider.Spec.BaseURL, APIKeys: []string{testAIProviderKey}, + }) + require.NoError(t, err) + got.Status.CreateAttemptTime = &metav1.Time{Time: time.Now().Add(-3 * time.Minute)} + require.NoError(t, k8sClient.Status().Update(ctx, got)) + got, _ = env.reconcileDeleting(ctx, t, got) + require.Nil(t, got, "the finalizer went") + require.Equal(t, 1, env.fake.requestCount(routeDeleteAIProvider)) + _, found := env.fake.aiProvider(provider.Name) + require.False(t, found, "the late create was deleted") +} + // A name lookup that answers this object's provider without an ID is // impossible. The reconcile fails fast, deletes nothing and keeps the // finalizer. While the control plane is being deleted, the one attempt is @@ -200,7 +233,7 @@ func testAIProviderDeleteAnswerWithoutID(t *testing.T, deleting bool) { require.NoError(t, err) provider.Finalizers = []string{coderv1alpha1.CoderAIProviderCleanupFinalizer} require.NoError(t, k8sClient.Update(ctx, provider)) - provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now()} + provider.Status.CreateAttemptTime = &metav1.Time{Time: time.Now().Add(-3 * time.Minute)} require.NoError(t, k8sClient.Status().Update(ctx, provider)) env.fake.failNext(routeAIProvider, fakeFault{Rewrite: func(answer any) any { p := answer.(codersdk.AIProvider) From 094f94886d6c1ed12872adb506a7ed4a0758c672 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 03:39:39 +0000 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=A4=96=20feat:=20activate=20the=20C?= =?UTF-8?q?oderAIProvider=20controller?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator now runs the CoderAIProvider controller and installs its CRD. The controller gets RBAC on coderaiproviders and its status and finalizers subresources. The Secret rule does not change. A new how-to covers the key Secret, the credential kill switch, the seeding limits and deletion. Refs #154 Change-Id: If0f7968062016829043a2559342ea3150a674f11 Signed-off-by: Thomas Kosiewski --- .cspell.json | 3 + .github/workflows/ci.yaml | 17 +- .../coder.com_coderaiproviders.yaml | 0 config/default/kustomization.yaml | 1 + config/rbac/role.yaml | 30 +- .../coder_v1alpha1_coderaiprovider.yaml | 25 ++ dist/install.yaml | 301 +++++++++++++++++- docs/how-to/manage-ai-providers.md | 155 +++++++++ docs/reference/api/coderaiprovider.md | 71 +++++ hack/update-manifests.sh | 2 +- internal/app/controllerapp/controllerapp.go | 8 + .../controller/coderaiprovider_controller.go | 13 +- internal/controller/coderaiprovider_env.go | 15 +- mkdocs.yml | 2 + 14 files changed, 601 insertions(+), 42 deletions(-) rename config/crd/{dormant => bases}/coder.com_coderaiproviders.yaml (100%) create mode 100644 config/samples/coder_v1alpha1_coderaiprovider.yaml create mode 100644 docs/how-to/manage-ai-providers.md create mode 100644 docs/reference/api/coderaiprovider.md diff --git a/.cspell.json b/.cspell.json index f00b59a65..a46b009c6 100644 --- a/.cspell.json +++ b/.cspell.json @@ -46,6 +46,9 @@ "codersdk", "codertemplate", "codertemplates", + "aibridge", + "coderaiprovider", + "coderaiproviders", "codertemplatetest", "codertemplatetests", "codertemplateversion", diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4eb88cf88..22ba36de0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -85,6 +85,7 @@ jobs: - 'internal/app/apiserverapp/**' - 'api/v1alpha1/**' - 'internal/controller/codertemplatetest*' + - 'internal/controller/coderaiprovider*' # Shared Coder access that CoderTemplateTest and later pool controllers call. - 'internal/controller/controlplane_access.go' - 'internal/controller/coder_access.go' @@ -155,22 +156,6 @@ jobs: exit 1 fi - - name: Verify the dormant CoderAIProvider CRD is not installed - # Remove this step when the CoderAIProvider controller ships (#154). - run: | - # grep exits 1 when nothing matches. Any other status (for example a missing file) fails the step. - status=0 - # The third pattern matches the CRD path entry in config/default/kustomization.yaml. - grep -n -E '^[[:space:]]+kind: CoderAIProvider$|coderaiproviders\.coder\.com|coder\.com_coderaiproviders\.yaml' \ - dist/install.yaml config/default/kustomization.yaml config/crd/bases/*.yaml || status=$? - if [[ "${status}" -eq 0 ]]; then - echo "assertion failed: the CoderAIProvider CRD must not ship before its controller" >&2 - exit 1 - elif [[ "${status}" -ne 1 ]]; then - echo "assertion failed: the CoderAIProvider CRD guard could not read the install manifests (grep exit ${status})" >&2 - exit 1 - fi - - name: Verify the dormant CoderAIGatewayKey CRD is not installed # Remove this step when the CoderAIGatewayKey controller ships (#156). run: | diff --git a/config/crd/dormant/coder.com_coderaiproviders.yaml b/config/crd/bases/coder.com_coderaiproviders.yaml similarity index 100% rename from config/crd/dormant/coder.com_coderaiproviders.yaml rename to config/crd/bases/coder.com_coderaiproviders.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 5d012eca2..a2042c150 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace-coder-system.yaml + - ../crd/bases/coder.com_coderaiproviders.yaml - ../crd/bases/coder.com_codercontrolplanes.yaml - ../crd/bases/coder.com_coderprovisioners.yaml - ../crd/bases/coder.com_codertemplatetests.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 33a5370f2..f1d9c5589 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -85,33 +85,49 @@ rules: - coder.com resources: - coderaigatewaykeys - - coderaiproviders - codermcpservers verbs: - list - apiGroups: - coder.com resources: - - codercontrolplanes - - coderprovisioners - - coderworkspaceproxies + - coderaiproviders verbs: - - create - - delete - get - list - patch - - update - watch - apiGroups: - coder.com resources: + - coderaiproviders/finalizers - codercontrolplanes/finalizers - coderprovisioners/finalizers - codertemplatetests/finalizers - coderworkspaceproxies/finalizers verbs: - update +- apiGroups: + - coder.com + resources: + - coderaiproviders/status + verbs: + - get + - update +- apiGroups: + - coder.com + resources: + - codercontrolplanes + - coderprovisioners + - coderworkspaceproxies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - coder.com resources: diff --git a/config/samples/coder_v1alpha1_coderaiprovider.yaml b/config/samples/coder_v1alpha1_coderaiprovider.yaml new file mode 100644 index 000000000..a762858f9 --- /dev/null +++ b/config/samples/coder_v1alpha1_coderaiprovider.yaml @@ -0,0 +1,25 @@ +# Manages one AI provider in Coder. The object name is the Coder provider +# name. See docs/how-to/manage-ai-providers.md. +# +# Prerequisites: +# - CoderControlPlane codercontrolplane-sample with status.operatorAccessReady +# true. +# - Secret openai-api-key in the same namespace, with the label +# coder.com/credential=true and the annotation +# coder.com/credential-destinations=https://api.openai.com. +apiVersion: coder.com/v1alpha1 +kind: CoderAIProvider +metadata: + name: openai + namespace: coder +spec: + controlPlaneRef: + name: codercontrolplane-sample + # Immutable: Coder cannot change the type after create. + type: openai + baseURL: https://api.openai.com/v1 + displayName: OpenAI + enabled: true + apiKeySecretRef: + name: openai-api-key + key: api-key diff --git a/dist/install.yaml b/dist/install.yaml index 4a2f1f0a6..b8e7b64ea 100644 --- a/dist/install.yaml +++ b/dist/install.yaml @@ -5,6 +5,277 @@ metadata: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: coderaiproviders.coder.com +spec: + group: coder.com + names: + kind: CoderAIProvider + listKind: CoderAIProviderList + plural: coderaiproviders + singular: coderaiprovider + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.type + name: Type + type: string + - jsonPath: .spec.enabled + name: Enabled + type: boolean + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + CoderAIProvider manages one AI provider in Coder. The object name is the + Coder provider name. It cannot be a UUID, because Coder looks up a + UUID-shaped name as a provider ID. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CoderAIProviderSpec defines one AI provider in Coder. + properties: + apiKeySecretRef: + description: |- + APIKeySecretRef references the key in a Secret in the same namespace + that holds the provider API key. The Secret must opt in to the + credential Secret contract. + properties: + key: + description: Key is the key inside the Secret data map. + type: string + name: + description: Name is the Kubernetes Secret name. + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: apiKeySecretRef.name must not be empty + rule: size(self.name) > 0 + - message: apiKeySecretRef.key must not be empty + rule: has(self.key) && size(self.key) > 0 + baseURL: + description: |- + BaseURL is the http:// or https:// URL that Coder sends model requests + to. It must not hold user info. The Secret in apiKeySecretRef must list + its origin as an allowed destination. + maxLength: 2048 + minLength: 1 + type: string + x-kubernetes-validations: + - message: baseURL must be a valid URL with a host + rule: isURL(self) && url(self).getHostname() != '' + - message: baseURL must be an http:// or https:// URL with a host + and no user info + rule: self.matches('^https?://[^/?#@\\s]+([/?#].*)?$') + controlPlaneRef: + description: |- + ControlPlaneRef names the CoderControlPlane in the same namespace. The + controller calls Coder with that control plane's operator token. + properties: + name: + description: Name is the CoderControlPlane name. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: controlPlaneRef is immutable + rule: self == oldSelf + displayName: + description: |- + DisplayName is the name that Coder shows. When empty, Coder shows the + provider name. + maxLength: 128 + type: string + enabled: + default: true + description: |- + Enabled turns the provider on or off in Coder. When omitted, the + default is true. + type: boolean + icon: + description: Icon is the icon URL or path that Coder shows. + maxLength: 2048 + type: string + type: + description: Type is the Coder provider type. Coder cannot change + it after create. + enum: + - openai + - anthropic + - azure + - google + - openai-compat + - openrouter + - vercel + type: string + x-kubernetes-validations: + - message: type is immutable + rule: self == oldSelf + required: + - apiKeySecretRef + - baseURL + - controlPlaneRef + - type + type: object + status: + description: CoderAIProviderStatus defines the observed state of a CoderAIProvider. + properties: + appliedGeneration: + description: AppliedGeneration is the last generation whose spec Coder + fully holds. + format: int64 + type: integer + conditions: + description: Conditions are Ready, Reconciling, and Stalled. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + createAttemptTime: + description: |- + CreateAttemptTime is set before every provider create request. While it + is set and ProviderID is empty, a create request can be in flight or + already committed in Coder. + format: date-time + type: string + credential: + description: Credential records the Secret value last sent to Coder. + properties: + keyID: + description: KeyID is the ID that Coder gave the key. It is not + the key. + type: string + resendAttemptTime: + description: |- + ResendAttemptTime is set before every key resend. While it is set, a + resend can be in flight or already committed in Coder. + format: date-time + type: string + secretKey: + description: SecretKey is the key in the Secret that was read. + type: string + secretName: + description: SecretName is the Secret name that was read. + type: string + secretResourceVersion: + description: SecretResourceVersion is the resourceVersion of the + Secret that was read. + type: string + secretUID: + description: SecretUID is the UID of the Secret that was read. + type: string + type: object + observedGeneration: + description: ObservedGeneration is the generation the controller last + processed. + format: int64 + type: integer + providerID: + description: ProviderID is the Coder provider ID. + type: string + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: 'metadata.name is the Coder provider name: at most 63 lowercase + letters, digits, and single hyphens between them' + rule: size(self.metadata.name) <= 63 && self.metadata.name.matches('^[a-z0-9]+(-[a-z0-9]+)*$') + - message: 'metadata.name must not be a UUID: Coder reads a UUID-shaped provider + name as a provider ID' + rule: '!self.metadata.name.matches(''^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'') + && !self.metadata.name.matches(''^[0-9a-f]{32}$'')' + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 @@ -5836,33 +6107,49 @@ rules: - coder.com resources: - coderaigatewaykeys - - coderaiproviders - codermcpservers verbs: - list - apiGroups: - coder.com resources: - - codercontrolplanes - - coderprovisioners - - coderworkspaceproxies + - coderaiproviders verbs: - - create - - delete - get - list - patch - - update - watch - apiGroups: - coder.com resources: + - coderaiproviders/finalizers - codercontrolplanes/finalizers - coderprovisioners/finalizers - codertemplatetests/finalizers - coderworkspaceproxies/finalizers verbs: - update +- apiGroups: + - coder.com + resources: + - coderaiproviders/status + verbs: + - get + - update +- apiGroups: + - coder.com + resources: + - codercontrolplanes + - coderprovisioners + - coderworkspaceproxies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - coder.com resources: diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md new file mode 100644 index 000000000..ba92f238e --- /dev/null +++ b/docs/how-to/manage-ai-providers.md @@ -0,0 +1,155 @@ +# Manage AI providers with CoderAIProvider + +A `CoderAIProvider` manages one AI provider in Coder (Coder's `/api/v2/ai/providers`). The operator creates the provider, keeps its settings and its API key equal to the spec, and deletes it when you delete the object. The object name is the Coder provider name. + +!!! warning "Who may write these resources" + The operator calls Coder as a site owner. Grant `create`, `update` and `patch` on `coderaiproviders` only to principals who may act as a Coder owner for AI providers in the whole deployment. Such a principal can use any credential Secret in the namespace that opted in, but only toward the origins that the Secret lists (see [Prepare the key Secret](#1-prepare-the-key-secret)). + +## Before you start + +- A `CoderControlPlane` in the same namespace with `status.operatorAccessReady: true`. The operator user must be a site owner. Coder answers `403` to anyone else on the provider routes. +- An API key for the provider, for example an OpenAI key. + +## 1. Prepare the key Secret + +The operator reads the key only from a Secret that opted in to the credential contract. The Secret needs both of these: + +- The label `coder.com/credential: "true"`. +- The annotation `coder.com/credential-destinations` with the origin of `spec.baseURL`, for example `https://api.openai.com`. An origin is `scheme://host`, plus `:port` for a port that is not the default. List only origins that you trust: an origin that redirects can pass the key on. + +```bash +kubectl -n coder create secret generic openai-api-key --from-literal=api-key="$OPENAI_API_KEY" +kubectl -n coder label secret openai-api-key coder.com/credential=true +kubectl -n coder annotate secret openai-api-key coder.com/credential-destinations=https://api.openai.com +``` + +The operator refuses the value when it is empty, larger than 8 KiB, starts or ends with whitespace (a trailing newline counts), is not valid UTF-8, or holds a control byte (for example NUL, CR or LF). It also refuses Secrets that the operator generates or that a `CoderControlPlane` uses, and Secrets with a type other than `Opaque`. [Configure credential Secrets](workspace-proxy-credentials.md#checks-the-operator-runs) lists every rule. + +A `baseURL` with plain `http://` is allowed for model servers inside the cluster, but Coder then sends the key in clear text. The Secret must list the `http://` origin. + +## 2. Create the provider + +```yaml +apiVersion: coder.com/v1alpha1 +kind: CoderAIProvider +metadata: + name: openai + namespace: coder +spec: + controlPlaneRef: + name: coder + type: openai + baseURL: https://api.openai.com/v1 + displayName: OpenAI + apiKeySecretRef: + name: openai-api-key + key: api-key +``` + +```bash +kubectl apply -f openai-provider.yaml +kubectl -n coder wait --for=condition=Ready coderaiprovider/openai --timeout=2m +``` + +`spec.type` and `spec.controlPlaneRef` cannot change. The name cannot be a UUID, because Coder reads a UUID-shaped name as a provider ID. The sample is in `config/samples/coder_v1alpha1_coderaiprovider.yaml`. + +The operator never takes over a provider that it did not create. When Coder already has a provider with the name, the object stalls with `NameConflict` and the operator sends no write. Choose another name, or delete that provider in Coder. + +To rotate the key, change the Secret value. The operator sends the new key with one update. A change of only the Secret's labels or annotations sends nothing. Coder's audit log records the masked key on every key write. + +## 3. Read the result + +```bash +kubectl -n coder get coderaiproviders +kubectl -n coder get coderaiprovider openai -o jsonpath='{range .status.conditions[*]}{.type}={.status} {.reason}: {.message}{"\n"}{end}' +``` + +The object has the conditions `Ready`, `Reconciling` and `Stalled`. A stalled object waits for a change of its spec or of an input. The most common reasons: + +| Reason | Meaning | +| --- | --- | +| `Applied` | Coder matches the spec (`Ready=True`). | +| `ControlPlaneNotReady`, `OperatorAccessNotReady` | The control plane cannot serve Coder calls yet. | +| `CoderUnavailable` | A Coder call failed. The operator retries every 30 seconds. | +| `CredentialSecretNotFound`, `CredentialNotOptedIn`, `CredentialDestinationNotAllowed`, `CredentialRefused`, `CredentialKeyMissing`, `CredentialMalformed` | The key Secret fails the credential contract. See [the credential kill switch](#the-credential-kill-switch). | +| `NameConflict` | Coder has a provider with this name that the operator did not create. | +| `NameReservedByEnv` | The control plane can seed a provider with this name. See [Seeding limits](#seeding-limits). | +| `CoderRejected` | Coder rejected the create request. Fix the spec. | +| `DisabledExternally`, `KeyChangedExternally`, `DeletedExternally` | Someone disabled the provider, changed its key or deleted it in Coder. See [Kill switches](#kill-switches). | + +## Kill switches + +### The credential kill switch + +When the key Secret fails the credential contract for a provider that the operator created, the operator disables the provider in Coder at once. This covers every credential error: the Secret is deleted, loses its label or the destinations annotation, no longer lists the origin that Coder uses, is refused, has no such key, or holds a malformed value. The update holds only `enabled: false`. Coder keeps the old key, but no longer uses it. + +The object stalls with the credential reason. Repairing the Secret does not enable the provider again. To enable it, fix the Secret, then edit the spec (any change of `spec`, for example `displayName`). A short read error of the Secret, such as an API server timeout, is not a credential error: the operator retries and changes nothing. + +### Dashboard changes + +A disable, a key change or a delete in the Coder dashboard is a kill switch too. The object stalls with `DisabledExternally`, `KeyChangedExternally` or `DeletedExternally`, and the operator does not undo the change until the spec changes. A spec edit then enables the provider, sends the Secret's key, or creates the provider again. + +To revoke a key in an emergency: + +1. Disable the provider in the Coder dashboard, or delete the key Secret or remove its label. +2. Revoke the key at the upstream provider. + +## Seeding limits + +Coder seeds AI providers at startup from its own settings, and it fails to start when a seeded provider differs from a provider with the same name in its database. So the operator does not create a provider whose name the control plane can seed. The object stalls with `NameReservedByEnv` and the message names the setting, never its value. + +The operator checks the control plane's `spec.extraEnv` and `spec.extraArgs`, in any letter case and with any value or `valueFrom`: + +| Setting | Reserved name | +| --- | --- | +| `CODER_AI_GATEWAY_OPENAI_KEY`, `CODER_AIBRIDGE_OPENAI_KEY`, or the matching `--ai-gateway-openai-key` and `--aibridge-openai-key` flags | `openai` | +| `CODER_AI_GATEWAY_ANTHROPIC_KEY`, `CODER_AIBRIDGE_ANTHROPIC_KEY`, any `CODER_AI_GATEWAY_BEDROCK_*` or `CODER_AIBRIDGE_BEDROCK_*` variable, and the matching flags | `anthropic` | +| A config file: `CODER_CONFIG_PATH`, `--config` or `-c`. Its Bedrock settings seed `anthropic`. The operator does not read the file. | `anthropic` | +| Any `CODER_AI_GATEWAY_PROVIDER__*` or `CODER_AIBRIDGE_PROVIDER__*` variable, or a `$(VAR)` reference in `spec.extraArgs` | every name: no provider is created | + +These limits remain: + +- The operator does not read `spec.envFrom` sources. A ConfigMap or Secret there that sets one of the variables above is not checked. Do not seed providers through `envFrom` on a control plane that `CoderAIProvider` objects use. +- The check runs only before a create. When you add a seeding setting after the provider exists, the control plane rolls coderd at once, and coderd fails to start if the seeded provider differs. Remove the setting, or delete the `CoderAIProvider` and create it with another name. + +## Delete a provider + +Deleting the object deletes the provider in Coder. With only an unconfirmed create, the operator reads Coder first and deletes only a provider that this object created. + +- When Coder is unreachable, or the control plane has no operator access, the finalizer `coder.com/ai-provider-cleanup` stays and the operator retries every 60 seconds. +- When the control plane is gone, nothing can delete the provider. The finalizer goes at once, and the operator logs the provider ID. Delete the provider in the Coder dashboard. +- While the control plane is being deleted, the operator sends one delete with a 10-second limit, then removes the finalizer. So a namespace deletion finishes. + +Delete `CoderAIProvider` objects before their control plane. Coder deletes a provider even while chat models use it. Those models stop working. + +To release an object by hand, for example when the control plane will never get operator access again: + +```bash +kubectl -n coder patch coderaiprovider openai --type=merge -p '{"metadata":{"finalizers":null}}' +``` + +## Grant access + +Each permission in the control plane's namespace maps to Coder power: + +| Permission | Coder power | +| --- | --- | +| `create`, `update`, `patch` on `coderaiproviders` | Coder owner for AI providers in the whole deployment, limited to the origins that opted-in Secrets list | +| `patch` on credential Secrets | Decides whether a key can be used (label) and where it can go (annotation) | +| `get`, `list` on `coderaiproviders` | Sees names, URLs, Secret names and Coder IDs. No secrets. | + +A viewer Role: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: coderaiprovider-viewer + namespace: coder +rules: + - apiGroups: ["coder.com"] + resources: ["coderaiproviders"] + verbs: ["get", "list", "watch"] +``` + +Give the editor verbs (`create`, `update`, `patch`, `delete`) only to subjects who may also read the credential Secrets. A `ResourceQuota` on `count/coderaiproviders.coder.com` limits how many providers a namespace can create. diff --git a/docs/reference/api/coderaiprovider.md b/docs/reference/api/coderaiprovider.md new file mode 100644 index 000000000..19cbf542d --- /dev/null +++ b/docs/reference/api/coderaiprovider.md @@ -0,0 +1,71 @@ + + +# `CoderAIProvider` + +## API identity + +- Group/version: `coder.com/v1alpha1` +- Kind: `CoderAIProvider` +- Resource: `coderaiproviders` +- Scope: namespaced + +## Spec + +| Field | Type | Description | +| --- | --- | --- | +| `controlPlaneRef` | [CoderControlPlaneReference](#codercontrolplanereference) | ControlPlaneRef names the CoderControlPlane in the same namespace. The controller calls Coder with that control plane's operator token. | +| `type` | string | Type is the Coder provider type. Coder cannot change it after create. | +| `baseURL` | string | BaseURL is the http:// or https:// URL that Coder sends model requests to. It must not hold user info. The Secret in apiKeySecretRef must list its origin as an allowed destination. | +| `displayName` | string | DisplayName is the name that Coder shows. When empty, Coder shows the provider name. | +| `icon` | string | Icon is the icon URL or path that Coder shows. | +| `enabled` | boolean | Enabled turns the provider on or off in Coder. When omitted, the default is true. | +| `apiKeySecretRef` | [SecretKeySelector](#secretkeyselector) | APIKeySecretRef references the key in a Secret in the same namespace that holds the provider API key. The Secret must opt in to the credential Secret contract. | + +## Status + +| Field | Type | Description | +| --- | --- | --- | +| `observedGeneration` | integer | ObservedGeneration is the generation the controller last processed. | +| `appliedGeneration` | integer | AppliedGeneration is the last generation whose spec Coder fully holds. | +| `providerID` | string | ProviderID is the Coder provider ID. | +| `createAttemptTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | CreateAttemptTime is set before every provider create request. While it is set and ProviderID is empty, a create request can be in flight or already committed in Coder. | +| `credential` | [CoderAIProviderCredentialStatus](#coderaiprovidercredentialstatus) | Credential records the Secret value last sent to Coder. | +| `conditions` | [Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array | Conditions are Ready, Reconciling, and Stalled. | + +## Referenced types + +### CoderAIProviderCredentialStatus + +CoderAIProviderCredentialStatus records which Secret value was last sent to +Coder. It holds no secret data and nothing derived from it. + +| Field | Type | Description | +| --- | --- | --- | +| `secretName` | string | SecretName is the Secret name that was read. | +| `secretKey` | string | SecretKey is the key in the Secret that was read. | +| `secretUID` | string | SecretUID is the UID of the Secret that was read. | +| `secretResourceVersion` | string | SecretResourceVersion is the resourceVersion of the Secret that was read. | +| `keyID` | string | KeyID is the ID that Coder gave the key. It is not the key. | +| `resendAttemptTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | ResendAttemptTime is set before every key resend. While it is set, a resend can be in flight or already committed in Coder. | + +### CoderControlPlaneReference + +CoderControlPlaneReference names a CoderControlPlane in the same namespace. + +| Field | Type | Description | +| --- | --- | --- | +| `name` | string | Name is the CoderControlPlane name. | + +### SecretKeySelector + +SecretKeySelector identifies a key in a Secret. + +| Field | Type | Description | +| --- | --- | --- | +| `name` | string | Name is the Kubernetes Secret name. | +| `key` | string | Key is the key inside the Secret data map. | + +## Source + +- Go type: `api/v1alpha1/coderaiprovider_types.go` +- Generated CRD: `config/crd/bases/coder.com_coderaiproviders.yaml` diff --git a/hack/update-manifests.sh b/hack/update-manifests.sh index 52fb281a4..a04309fc8 100755 --- a/hack/update-manifests.sh +++ b/hack/update-manifests.sh @@ -31,7 +31,7 @@ list_manifests() { # CRDs of kinds whose controller is not registered yet. They go to config/crd/dormant/, which no kustomization, # example, doc or script applies, so the kind is never served without its controller. # Move a file back by deleting its entry when the controller ships. -DORMANT_CRDS=(coder.com_coderaiproviders.yaml coder.com_coderaigatewaykeys.yaml coder.com_codermcpservers.yaml coder.com_coderworkspacepools.yaml coder.com_coderworkspacepoolmembers.yaml coder.com_coderworkspaceclaims.yaml) +DORMANT_CRDS=(coder.com_coderaigatewaykeys.yaml coder.com_codermcpservers.yaml coder.com_coderworkspacepools.yaml coder.com_coderworkspacepoolmembers.yaml coder.com_coderworkspaceclaims.yaml) # RBAC manifests that only the aggregated API server needs; the controller-only install bundle leaves them out. BUNDLE_EXCLUDED_RBAC=(apiservice-cabundle-role.yaml auth-delegator-binding.yaml authentication-reader-binding.yaml) diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index a1aaa13b0..360be7674 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -126,6 +126,14 @@ func SetupControllers(mgr manager.Manager) error { return fmt.Errorf("unable to create template test controller: %w", err) } + aiProviderReconciler := &controller.CoderAIProviderReconciler{ + Client: client, + Scheme: managerScheme, + } + if err := aiProviderReconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create AI provider controller: %w", err) + } + return nil } diff --git a/internal/controller/coderaiprovider_controller.go b/internal/controller/coderaiprovider_controller.go index b23d8c724..914fec42f 100644 --- a/internal/controller/coderaiprovider_controller.go +++ b/internal/controller/coderaiprovider_controller.go @@ -39,8 +39,12 @@ const ( aiProviderRetry = 30 * time.Second ) -// CoderAIProviderReconciler reconciles CoderAIProvider objects (#154). It is -// not registered yet. It checks the spec, the credential Secret and the +// +kubebuilder:rbac:groups=coder.com,resources=coderaiproviders,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=coder.com,resources=coderaiproviders/status,verbs=get;update +// +kubebuilder:rbac:groups=coder.com,resources=coderaiproviders/finalizers,verbs=update + +// CoderAIProviderReconciler reconciles CoderAIProvider objects (#154). It +// checks the spec, the credential Secret and the // control plane, creates the Coder provider when the name is free, and sets // the spec values on the provider that it created. It never changes a // provider that it did not create. @@ -70,9 +74,8 @@ func aiProviderStalled(requeue time.Duration, reason, format string, args ...any return &aiProviderStep{stalled: true, reason: reason, message: fmt.Sprintf(format, args...), requeue: requeue} } -// SetupWithManager wires the reconciler into controller-runtime. Nothing -// calls it until activation. A change of the referenced control plane or of -// the key Secret requeues the provider. +// SetupWithManager wires the reconciler into controller-runtime. A change of +// the referenced control plane or of the key Secret requeues the provider. func (r *CoderAIProviderReconciler) SetupWithManager(mgr ctrl.Manager) error { if mgr == nil || r.Client == nil || r.Scheme == nil { return fmt.Errorf("assertion failed: AI provider reconciler needs a manager, a client, and a scheme") diff --git a/internal/controller/coderaiprovider_env.go b/internal/controller/coderaiprovider_env.go index 33ea59859..24c8832b7 100644 --- a/internal/controller/coderaiprovider_env.go +++ b/internal/controller/coderaiprovider_env.go @@ -37,12 +37,15 @@ var ( // CODER_..._PROVIDER__* configures an indexed provider, whose name // any of its variables can set. These have no flags. aiProviderIndexedEnvPrefixes = []string{"CODER_AI_GATEWAY_PROVIDER_", "CODER_AIBRIDGE_PROVIDER_"} - // A config file can set the YAML keys bedrock_base_url, bedrock_region, - // bedrock_model and bedrock_small_fast_model, and each one seeds - // anthropic (codersdk/deployment.go:1968-2026, :4493-4568 and - // coderd/ai_providers_migrate.go:321-357). Keys and indexed providers - // have no YAML keys (cli/server.go:994), so a config file seeds no other - // name. The file is not read: the check fails closed (#154). + // A config file can set the YAML keys bedrock_base_url and + // bedrock_region, and either one seeds anthropic + // (codersdk/deployment.go:1968-1986, :4493-4514, + // codersdk/aiproviders_bedrock.go:82-96 and + // coderd/ai_providers_migrate.go:321-357). bedrock_model and + // bedrock_small_fast_model alone seed nothing. Keys and indexed + // providers have no YAML keys (cli/server.go:994), so a config file + // seeds no other name. The file is not read: the check fails closed + // (#154). aiProviderConfigFileEnv = "CODER_CONFIG_PATH" aiProviderConfigFileFlag = "config" ) diff --git a/mkdocs.yml b/mkdocs.yml index bdb9e8abb..8e80530ae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - Configure credential Secrets for proxies and provisioners: how-to/workspace-proxy-credentials.md - Start, stop, and read logs of workspaces: how-to/start-stop-workspaces.md - Test a template version: how-to/test-templates.md + - Manage AI providers: how-to/manage-ai-providers.md - Gate template promotion with GitOps: how-to/gitops.md - Run the MCP server: how-to/mcp-server.md - Troubleshooting: how-to/troubleshooting.md @@ -50,6 +51,7 @@ nav: - Aggregated API behavior: reference/aggregated-api-behavior.md - API: # BEGIN GENERATED API NAV + - CoderAIProvider: reference/api/coderaiprovider.md - CoderControlPlane: reference/api/codercontrolplane.md - CoderProvisioner: reference/api/coderprovisioner.md - CoderTemplateTest: reference/api/codertemplatetest.md From a75623fba08596efcb754f7ab37374c9036cc409 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:00:07 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=A4=96=20docs:=20delete=20CoderAIPr?= =?UTF-8?q?oviders=20before=20their=20control=20plane,=20and=20treat=20del?= =?UTF-8?q?ete=20as=20owner=20power?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: Idfa61c5e1d811b02075dca247895563b9fb1242d Signed-off-by: Thomas Kosiewski --- docs/how-to/manage-ai-providers.md | 5 +++-- docs/tutorials/getting-started.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md index ba92f238e..7ad18fa71 100644 --- a/docs/how-to/manage-ai-providers.md +++ b/docs/how-to/manage-ai-providers.md @@ -3,7 +3,7 @@ A `CoderAIProvider` manages one AI provider in Coder (Coder's `/api/v2/ai/providers`). The operator creates the provider, keeps its settings and its API key equal to the spec, and deletes it when you delete the object. The object name is the Coder provider name. !!! warning "Who may write these resources" - The operator calls Coder as a site owner. Grant `create`, `update` and `patch` on `coderaiproviders` only to principals who may act as a Coder owner for AI providers in the whole deployment. Such a principal can use any credential Secret in the namespace that opted in, but only toward the origins that the Secret lists (see [Prepare the key Secret](#1-prepare-the-key-secret)). + The operator calls Coder as a site owner. Grant `create`, `update`, `patch` and `delete` on `coderaiproviders` only to principals who may act as a Coder owner for AI providers in the whole deployment. A delete makes the operator delete the provider in Coder. Such a principal can use any credential Secret in the namespace that opted in, but only toward the origins that the Secret lists (see [Prepare the key Secret](#1-prepare-the-key-secret)). ## Before you start @@ -135,6 +135,7 @@ Each permission in the control plane's namespace maps to Coder power: | Permission | Coder power | | --- | --- | | `create`, `update`, `patch` on `coderaiproviders` | Coder owner for AI providers in the whole deployment, limited to the origins that opted-in Secrets list | +| `delete` on `coderaiproviders` | Deletes the provider in Coder, for every user of the deployment | | `patch` on credential Secrets | Decides whether a key can be used (label) and where it can go (annotation) | | `get`, `list` on `coderaiproviders` | Sees names, URLs, Secret names and Coder IDs. No secrets. | @@ -152,4 +153,4 @@ rules: verbs: ["get", "list", "watch"] ``` -Give the editor verbs (`create`, `update`, `patch`, `delete`) only to subjects who may also read the credential Secrets. A `ResourceQuota` on `count/coderaiproviders.coder.com` limits how many providers a namespace can create. +Give the editor verbs (`create`, `update`, `patch`, `delete`) only to subjects who may act as a Coder owner for AI providers and may also read the credential Secrets. A `ResourceQuota` on `count/coderaiproviders.coder.com` limits how many providers a namespace can create. diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md index 36494be72..5d76621c5 100644 --- a/docs/tutorials/getting-started.md +++ b/docs/tutorials/getting-started.md @@ -63,14 +63,15 @@ Then open `http://127.0.0.1:3000` in a browser. ## 5. Clean up (optional) -Delete the control plane first, while the operator still runs. The operator then cleans up and removes its finalizer. After that, remove the bundle. The commands set `BASE` again, because you can be in a new shell. +Delete the `CoderAIProvider` objects first, if you created any, and then the control plane, while the operator still runs. The operator then deletes the providers in Coder, cleans up, and removes its finalizers. A provider deleted after its control plane stays in Coder. After that, remove the bundle. The commands set `BASE` again, because you can be in a new shell. !!! warning - Delete all other `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` resources before you delete the bundle. When you delete the bundle, you also delete the `coder.com` CRDs, and Kubernetes then removes all remaining resources of these kinds in the cluster. When you delete the `coder` namespace, you also delete all other objects in it, for example Secrets and PersistentVolumeClaims. + Delete all other `CoderAIProvider`, `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` resources before you delete the bundle. Delete `CoderAIProvider` resources before their control plane. When you delete the bundle, you also delete the `coder.com` CRDs, and Kubernetes then removes all remaining resources of these kinds in the cluster. When you delete the `coder` namespace, you also delete all other objects in it, for example Secrets and PersistentVolumeClaims. ```bash BASE="https://raw.githubusercontent.com/coder/coder-k8s/main" +kubectl delete coderaiproviders --all -n coder --wait --timeout=120s kubectl delete -f "$BASE/config/samples/coder_v1alpha1_codercontrolplane.yaml" --ignore-not-found kubectl wait --for=delete codercontrolplane/codercontrolplane-sample -n coder --timeout=120s kubectl delete -f "$BASE/dist/install.yaml" --ignore-not-found From 0e343ff1aa210d1d40b7eeb9435dc97361de2a2b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:07:57 +0000 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=A4=96=20docs:=20say=20when=20the?= =?UTF-8?q?=20operator=20notices=20a=20dashboard=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I7ba108fd9674446d12ecb8f4f0aee6e3e5f5624b Signed-off-by: Thomas Kosiewski --- docs/how-to/manage-ai-providers.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md index 7ad18fa71..e24680be3 100644 --- a/docs/how-to/manage-ai-providers.md +++ b/docs/how-to/manage-ai-providers.md @@ -89,6 +89,8 @@ The object stalls with the credential reason. Repairing the Secret does not enab A disable, a key change or a delete in the Coder dashboard is a kill switch too. The object stalls with `DisabledExternally`, `KeyChangedExternally` or `DeletedExternally`, and the operator does not undo the change until the spec changes. A spec edit then enables the provider, sends the Secret's key, or creates the provider again. +Coder sends no events, so the operator notices a dashboard change at its next read of the provider: at the resync, about every 10 minutes, or at once when the object, its key Secret or its control plane changes. Until then the object can still show `Ready=True`. + To revoke a key in an emergency: 1. Disable the provider in the Coder dashboard, or delete the key Secret or remove its label. From bc3ce276359261c95bc82859a9cd620cb66e5259 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:26:08 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=F0=9F=A4=96=20docs:=20qualify=20the=20me?= =?UTF-8?q?tadata-only=20and=20bounded-delete=20notes,=20and=20list=20Code?= =?UTF-8?q?rAIProvider=20in=20the=20inventories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I941c32cbe8915d7cd6bfeeca63908111226e6ad5 Signed-off-by: Thomas Kosiewski --- docs/explanation/architecture.md | 4 ++-- docs/how-to/deploy-controller.md | 2 +- docs/how-to/manage-ai-providers.md | 4 ++-- docs/index.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 51379e013..75a4d256c 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -31,7 +31,7 @@ graph TD | --- | --- | --- | --- | | Code | `internal/app/controllerapp/`, `internal/controller/` | `internal/app/apiserverapp/`, `internal/aggregated/storage/`, `internal/aggregated/coder/` | `internal/app/mcpapp/` | | Listens on | `:8081` (`/healthz`, `/readyz`) | `:6443` HTTPS (default) | `127.0.0.1:8090` only. `/mcp` requires the bearer token. `/healthz` and `/readyz` do not. | -| Resources | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy` | `coderworkspaces`, `codertemplates`, `codertemplateversions` | Tools for control planes, templates, workspaces, events, pod logs, and run state | +| Resources | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider` | `coderworkspaces`, `codertemplates`, `codertemplateversions` | Tools for control planes, templates, workspaces, events, pod logs, and run state | ### Controller @@ -75,7 +75,7 @@ graph TD | Path | Contents | | --- | --- | -| `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` | +| `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, and `CoderAIProvider` | | `config/rbac/` | The `coder-k8s` ServiceAccount, the `manager-role` ClusterRole, and the bindings (including auth-delegator) | | `config/apiserver-standalone/` | For standalone `--app=aggregated-apiserver`: the `coder-k8s-apiserver` ServiceAccount, its RBAC, and the placeholder Secret for the serving CA | | `deploy/deployment.yaml` | The `coder-k8s` Deployment (default `--app=all`) | diff --git a/docs/how-to/deploy-controller.md b/docs/how-to/deploy-controller.md index fb9bc5086..d83ac64f3 100644 --- a/docs/how-to/deploy-controller.md +++ b/docs/how-to/deploy-controller.md @@ -1,6 +1,6 @@ # Deploy the controller -Run `coder-k8s` as an operator only (`--app=controller`). The operator reconciles `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, and `CoderTemplateTest` resources. +Run `coder-k8s` as an operator only (`--app=controller`). The operator reconciles `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, and `CoderAIProvider` resources. Run the commands from a clone of this repository. diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md index e24680be3..10da24b34 100644 --- a/docs/how-to/manage-ai-providers.md +++ b/docs/how-to/manage-ai-providers.md @@ -55,7 +55,7 @@ kubectl -n coder wait --for=condition=Ready coderaiprovider/openai --timeout=2m The operator never takes over a provider that it did not create. When Coder already has a provider with the name, the object stalls with `NameConflict` and the operator sends no write. Choose another name, or delete that provider in Coder. -To rotate the key, change the Secret value. The operator sends the new key with one update. A change of only the Secret's labels or annotations sends nothing. Coder's audit log records the masked key on every key write. +To rotate the key, change the Secret value. The operator sends the new key with one update. A change of only the Secret's labels or annotations sends no key, as long as the Secret keeps its consent for the origin: removing the label or the origin is the [credential kill switch](#the-credential-kill-switch). The operator remembers the sent key in memory only, so after an operator restart the first change of the Secret sends the key once more. Coder's audit log records the masked key on every key write. ## 3. Read the result @@ -120,7 +120,7 @@ Deleting the object deletes the provider in Coder. With only an unconfirmed crea - When Coder is unreachable, or the control plane has no operator access, the finalizer `coder.com/ai-provider-cleanup` stays and the operator retries every 60 seconds. - When the control plane is gone, nothing can delete the provider. The finalizer goes at once, and the operator logs the provider ID. Delete the provider in the Coder dashboard. -- While the control plane is being deleted, the operator sends one delete with a 10-second limit, then removes the finalizer. So a namespace deletion finishes. +- While the control plane is being deleted and its operator access still works, the operator sends one delete with a 10-second limit, then removes the finalizer. Without working access, the finalizer goes without a delete. Either way a namespace deletion finishes. When the delete did not succeed, the operator logs the provider ID: delete the provider in the Coder dashboard. Delete `CoderAIProvider` objects before their control plane. Coder deletes a provider even while chat models use it. Those models stop working. diff --git a/docs/index.md b/docs/index.md index c32d3a9d6..c82d1a7e7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Use native Kubernetes APIs to run and manage [Coder](https://coder.com). | Component | Resources | | --- | --- | -| Operator | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy` (`coder.com/v1alpha1`) | +| Operator | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider` (`coder.com/v1alpha1`) | | Aggregated API server | `CoderWorkspace`, `CoderTemplate`, `CoderTemplateVersion` (`aggregation.coder.com/v1alpha1`) | | MCP server | Operational tools over HTTP | From 7b55c2765f47b132cb746d9869d33f8d34218659 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 04:39:19 +0000 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=A4=96=20docs:=20note=20the=20creat?= =?UTF-8?q?e=20settle=20window=20on=20delete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I197dbdca709523bdd0a4f7bffe3e5690408be8b1 Signed-off-by: Thomas Kosiewski --- docs/how-to/manage-ai-providers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md index 10da24b34..f3e284b09 100644 --- a/docs/how-to/manage-ai-providers.md +++ b/docs/how-to/manage-ai-providers.md @@ -116,7 +116,7 @@ These limits remain: ## Delete a provider -Deleting the object deletes the provider in Coder. With only an unconfirmed create, the operator reads Coder first and deletes only a provider that this object created. +Deleting the object deletes the provider in Coder. With only an unconfirmed create, the operator reads Coder first and deletes only a provider that this object created. A create request can still commit for a short time, so within 2 minutes of that request the operator reads again before it trusts that the provider is absent. - When Coder is unreachable, or the control plane has no operator access, the finalizer `coder.com/ai-provider-cleanup` stays and the operator retries every 60 seconds. - When the control plane is gone, nothing can delete the provider. The finalizer goes at once, and the operator logs the provider ID. Delete the provider in the Coder dashboard. From 885428e2e8e4404dd2e105c02785ffd6c66f3578 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 05:32:51 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=A4=96=20docs:=20add=20a=20stuck=20?= =?UTF-8?q?finalizer=20section=20for=20CoderAIProvider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I90d1eedc61dc0fb679a738da32a7b1db88e1ac8b Signed-off-by: Thomas Kosiewski --- docs/how-to/manage-ai-providers.md | 35 +++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md index f3e284b09..b06853140 100644 --- a/docs/how-to/manage-ai-providers.md +++ b/docs/how-to/manage-ai-providers.md @@ -118,17 +118,42 @@ These limits remain: Deleting the object deletes the provider in Coder. With only an unconfirmed create, the operator reads Coder first and deletes only a provider that this object created. A create request can still commit for a short time, so within 2 minutes of that request the operator reads again before it trusts that the provider is absent. -- When Coder is unreachable, or the control plane has no operator access, the finalizer `coder.com/ai-provider-cleanup` stays and the operator retries every 60 seconds. +- When Coder is unreachable, or the control plane has no operator access, the finalizer `coder.com/ai-provider-cleanup` stays and the operator retries every 60 seconds. See [Stuck finalizer](#stuck-finalizer). - When the control plane is gone, nothing can delete the provider. The finalizer goes at once, and the operator logs the provider ID. Delete the provider in the Coder dashboard. - While the control plane is being deleted and its operator access still works, the operator sends one delete with a 10-second limit, then removes the finalizer. Without working access, the finalizer goes without a delete. Either way a namespace deletion finishes. When the delete did not succeed, the operator logs the provider ID: delete the provider in the Coder dashboard. Delete `CoderAIProvider` objects before their control plane. Coder deletes a provider even while chat models use it. Those models stop working. -To release an object by hand, for example when the control plane will never get operator access again: +### Stuck finalizer -```bash -kubectl -n coder patch coderaiprovider openai --type=merge -p '{"metadata":{"finalizers":null}}' -``` +A deleted `CoderAIProvider` stays `Terminating` while the operator cannot prove that Coder no longer holds its provider. Two cases never end by themselves: + +- **An endless `CoderUnavailable` wait.** The `Ready` condition message starts with `delete provider ` or `get provider `, and the operator retries every 60 seconds. Coder is unreachable, or the delete got a 404 that Coder did not send, for example an HTML page from a proxy or `Route not found.` from a wrong URL. The operator trusts only Coder's own answer `Resource not found or you do not have access to this resource`. +- **An `assertion failed:` error.** `status.providerID` is not a UUID, so the operator sends no request to Coder. The status does not change. The error is in the operator log: + + ```bash + kubectl -n coder-system logs deploy/coder-k8s | grep -F 'assertion failed: AI provider coder/openai' + ``` + +Fix the cause if you can (Coder access, the proxy, the base URL), and the next retry deletes the provider. Otherwise release the object by hand: + +1. Note the provider name (the object name) and the provider ID from the condition message or from the `ControlPlaneGone` and `assertion failed:` log lines. After an `assertion failed:` error, do not trust `status.providerID`. +2. Find the index of the finalizer: + + ```bash + kubectl -n coder get coderaiprovider openai -o jsonpath='{.metadata.finalizers}{"\n"}' + ``` + +3. Remove it with a JSON patch. The `test` operation makes the patch fail, with no change, if another finalizer is at that index. Replace `0` with the index from step 2: + + ```bash + kubectl -n coder patch coderaiprovider openai --type=json -p '[ + {"op": "test", "path": "/metadata/finalizers/0", "value": "coder.com/ai-provider-cleanup"}, + {"op": "remove", "path": "/metadata/finalizers/0"} + ]' + ``` + +4. Check Coder for a leftover provider: open **AI settings > Providers** in the dashboard (`/ai/settings/providers`) as an owner, and look for the provider name. If it is there and nothing else uses it, delete it in the dashboard. ## Grant access