diff --git a/.cspell.json b/.cspell.json index a46b009c6..df441da00 100644 --- a/.cspell.json +++ b/.cspell.json @@ -47,6 +47,10 @@ "codertemplate", "codertemplates", "aibridge", + "aigateway", + "stakater", + "coderaigatewaykey", + "coderaigatewaykeys", "coderaiprovider", "coderaiproviders", "codertemplatetest", diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22ba36de0..645ec1401 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,6 +86,9 @@ jobs: - 'api/v1alpha1/**' - 'internal/controller/codertemplatetest*' - 'internal/controller/coderaiprovider*' + - 'internal/controller/coderaigatewaykey*' + - 'internal/controller/owned_secret.go' + - 'internal/controller/reserved_secret_names.go' # Shared Coder access that CoderTemplateTest and later pool controllers call. - 'internal/controller/controlplane_access.go' - 'internal/controller/coder_access.go' @@ -156,22 +159,6 @@ jobs: exit 1 fi - - name: Verify the dormant CoderAIGatewayKey CRD is not installed - # Remove this step when the CoderAIGatewayKey controller ships (#156). - run: | - # grep exits 1 when nothing matches. Any other status (for example a missing file) fails the step. - status=0 - # The third pattern matches the CRD path entry in config/default/kustomization.yaml. - grep -n -E '^[[:space:]]+kind: CoderAIGatewayKey$|coderaigatewaykeys\.coder\.com|coder\.com_coderaigatewaykeys\.yaml' \ - dist/install.yaml config/default/kustomization.yaml config/crd/bases/*.yaml || status=$? - if [[ "${status}" -eq 0 ]]; then - echo "assertion failed: the CoderAIGatewayKey CRD must not ship before its controller" >&2 - exit 1 - elif [[ "${status}" -ne 1 ]]; then - echo "assertion failed: the CoderAIGatewayKey CRD guard could not read the install manifests (grep exit ${status})" >&2 - exit 1 - fi - - name: Verify the dormant CoderMCPServer CRD is not installed # Remove this step when the CoderMCPServer controller ships (#155). run: | diff --git a/api/v1alpha1/coderaigatewaykey_types.go b/api/v1alpha1/coderaigatewaykey_types.go index 5a09d63d8..4eea40904 100644 --- a/api/v1alpha1/coderaigatewaykey_types.go +++ b/api/v1alpha1/coderaigatewaykey_types.go @@ -36,9 +36,6 @@ type AIGatewaySpec struct { // controller creates no key. Setting it back to false stops new keys and // rotations. The current key of each CoderAIGatewayKey stays until that // object is deleted. - // - // This field has no effect yet: the CoderAIGatewayKey controller is not - // shipped. // +optional AllowKeys bool `json:"allowKeys,omitempty"` } @@ -237,8 +234,12 @@ type CoderAIGatewayKeyStatus struct { // CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway // replicas and stores it in a Secret that this object owns. A gateway key // lets its holder load the API key of every enabled AI provider, so write -// access to this kind is admin-only. Deleting the object revokes every key -// that it created at once, and gateways that use them stop within a minute. +// access to this kind is admin-only. Deleting the object deletes every key +// that it created in Coder: after the 5-minute settle window of its last key +// create, and only while the control plane can reach Coder. A failed +// deletion keeps the object and retries. When the control plane is gone, the +// object goes and its keys stay valid in Coder. A gateway stops within a +// minute after its key is deleted. type CoderAIGatewayKey struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/codercontrolplane_types.go b/api/v1alpha1/codercontrolplane_types.go index 3f20088ea..13c2cfae7 100644 --- a/api/v1alpha1/codercontrolplane_types.go +++ b/api/v1alpha1/codercontrolplane_types.go @@ -132,8 +132,7 @@ type CoderControlPlaneSpec struct { MCPServers *MCPServersSpec `json:"mcpServers,omitempty"` // AIGateway configures CoderAIGatewayKey objects that use this control - // plane. It has no effect yet: the CoderAIGatewayKey controller is not - // shipped. + // plane. // +optional AIGateway *AIGatewaySpec `json:"aiGateway,omitempty"` diff --git a/config/crd/dormant/coder.com_coderaigatewaykeys.yaml b/config/crd/bases/coder.com_coderaigatewaykeys.yaml similarity index 97% rename from config/crd/dormant/coder.com_coderaigatewaykeys.yaml rename to config/crd/bases/coder.com_coderaigatewaykeys.yaml index a7dca985b..bd0d3c41b 100644 --- a/config/crd/dormant/coder.com_coderaigatewaykeys.yaml +++ b/config/crd/bases/coder.com_coderaigatewaykeys.yaml @@ -37,8 +37,12 @@ spec: CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway replicas and stores it in a Secret that this object owns. A gateway key lets its holder load the API key of every enabled AI provider, so write - access to this kind is admin-only. Deleting the object revokes every key - that it created at once, and gateways that use them stop within a minute. + access to this kind is admin-only. Deleting the object deletes every key + that it created in Coder: after the 5-minute settle window of its last key + create, and only while the control plane can reach Coder. A failed + deletion keeps the object and retries. When the control plane is gone, the + object goes and its keys stay valid in Coder. A gateway stops within a + minute after its key is deleted. properties: apiVersion: description: |- diff --git a/config/crd/bases/coder.com_codercontrolplanes.yaml b/config/crd/bases/coder.com_codercontrolplanes.yaml index 87d0c56cd..a0953b7c7 100644 --- a/config/crd/bases/coder.com_codercontrolplanes.yaml +++ b/config/crd/bases/coder.com_codercontrolplanes.yaml @@ -954,8 +954,7 @@ spec: aiGateway: description: |- AIGateway configures CoderAIGatewayKey objects that use this control - plane. It has no effect yet: the CoderAIGatewayKey controller is not - shipped. + plane. properties: allowKeys: description: |- @@ -967,9 +966,6 @@ spec: controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. - - This field has no effect yet: the CoderAIGatewayKey controller is not - shipped. type: boolean type: object certs: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index a2042c150..d3f9b0e3a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace-coder-system.yaml + - ../crd/bases/coder.com_coderaigatewaykeys.yaml - ../crd/bases/coder.com_coderaiproviders.yaml - ../crd/bases/coder.com_codercontrolplanes.yaml - ../crd/bases/coder.com_coderprovisioners.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f1d9c5589..561f059eb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -85,12 +85,6 @@ rules: - coder.com resources: - coderaigatewaykeys - - codermcpservers - verbs: - - list -- apiGroups: - - coder.com - resources: - coderaiproviders verbs: - get @@ -100,6 +94,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/finalizers - coderaiproviders/finalizers - codercontrolplanes/finalizers - coderprovisioners/finalizers @@ -110,6 +105,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/status - coderaiproviders/status verbs: - get @@ -139,6 +135,12 @@ rules: - get - patch - update +- apiGroups: + - coder.com + resources: + - codermcpservers + verbs: + - list - apiGroups: - coder.com resources: diff --git a/config/samples/coder_v1alpha1_coderaigatewaykey.yaml b/config/samples/coder_v1alpha1_coderaigatewaykey.yaml new file mode 100644 index 000000000..67b859e3a --- /dev/null +++ b/config/samples/coder_v1alpha1_coderaigatewaykey.yaml @@ -0,0 +1,24 @@ +# Keeps one Coder AI Gateway key in an owned Secret and rotates it. See +# docs/how-to/manage-ai-gateway-keys.md. +# +# Prerequisites: +# - A Coder Premium license with the AI Gateway enabled. +# - CoderControlPlane codercontrolplane-sample with status.operatorAccessReady +# true and spec.aiGateway.allowKeys true. +# - No Secret named coder-ai-gateway-key in the namespace: the operator +# creates it. +apiVersion: coder.com/v1alpha1 +kind: CoderAIGatewayKey +metadata: + name: gateway + namespace: coder +spec: + # Immutable. + controlPlaneRef: + name: codercontrolplane-sample + # Immutable. The key goes into data key "key". + secretName: coder-ai-gateway-key + rotation: + intervalSeconds: 2592000 + gracePeriodSeconds: 3600 + consumerWaitSeconds: 86400 diff --git a/dist/install.yaml b/dist/install.yaml index b8e7b64ea..8138d4996 100644 --- a/dist/install.yaml +++ b/dist/install.yaml @@ -5,6 +5,393 @@ metadata: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: coderaigatewaykeys.coder.com +spec: + group: coder.com + names: + kind: CoderAIGatewayKey + listKind: CoderAIGatewayKeyList + plural: coderaigatewaykeys + singular: coderaigatewaykey + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.secretName + name: Secret + type: string + - jsonPath: .status.currentKey.name + name: Current + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.nextRotationTime + name: Next rotation + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + CoderAIGatewayKey manages a Coder AI Gateway key for standalone AI Gateway + replicas and stores it in a Secret that this object owns. A gateway key + lets its holder load the API key of every enabled AI provider, so write + access to this kind is admin-only. Deleting the object deletes every key + that it created in Coder: after the 5-minute settle window of its last key + create, and only while the control plane can reach Coder. A failed + deletion keeps the object and retries. When the control plane is gone, the + object goes and its keys stay valid in Coder. A gateway stops within a + minute after its key is deleted. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CoderAIGatewayKeySpec defines a Coder AI Gateway key and + its rotation. + properties: + controlPlaneRef: + description: |- + ControlPlaneRef names the CoderControlPlane in the same namespace. The + controller creates and deletes keys with that control plane's operator + token. The control plane must allow keys in spec.aiGateway.allowKeys. + properties: + name: + description: Name is the CoderControlPlane name. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: controlPlaneRef is immutable + rule: self == oldSelf + rotation: + default: {} + description: |- + Rotation configures when the controller replaces the key and how long a + replaced key stays valid. + properties: + consumerWaitSeconds: + default: 86400 + description: |- + ConsumerWaitSeconds is how long after the grace period the controller + keeps a replaced key while Coder still reports connections that use it. + After that time the controller deletes the key, and gateways that still + use it exit and restart with the key from the Secret. 0 deletes the key + when the grace period ends. At most 7 days, default 1 day. + format: int32 + maximum: 604800 + minimum: 0 + type: integer + gracePeriodSeconds: + default: 3600 + description: |- + GracePeriodSeconds is the minimum time between writing a new key to the + Secret and deleting the replaced key in Coder. Consumers must reload + within it. Between 10 minutes and 7 days, default 1 hour. + format: int32 + maximum: 604800 + minimum: 600 + type: integer + intervalSeconds: + default: 2592000 + description: |- + IntervalSeconds is the time between scheduled rotations, counted from + the time the current key was written to the Secret. 0 turns scheduled + rotation off. Otherwise it is between 1 day and 365 days. The default is + 30 days, because Coder keys never expire. + format: int32 + maximum: 31536000 + minimum: 0 + type: integer + x-kubernetes-validations: + - message: intervalSeconds must be 0 or at least 86400 (1 day) + rule: self == 0 || self >= 86400 + trigger: + description: |- + Trigger starts one rotation each time it changes to a new non-empty + value. The value set when the object is created starts no rotation. + maxLength: 63 + type: string + type: object + x-kubernetes-validations: + - message: gracePeriodSeconds must be shorter than intervalSeconds + rule: '!has(self.intervalSeconds) || !has(self.gracePeriodSeconds) + || self.intervalSeconds == 0 || self.gracePeriodSeconds < self.intervalSeconds' + secretName: + description: |- + SecretName is the Secret in the same namespace that receives the key, + under the data key "key". The controller creates the Secret and owns it. + It never writes a Secret that already exists and that this object does + not own. To move the key to another Secret, delete and re-create this + object. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: secretName is immutable + rule: self == oldSelf + required: + - controlPlaneRef + - secretName + type: object + status: + description: |- + CoderAIGatewayKeyStatus defines the observed state of a CoderAIGatewayKey. + It holds Coder key IDs, names, serials, and times only. It never holds a + key, a key prefix, a hash, or a masked key. + properties: + answerMismatchGeneration: + description: |- + AnswerMismatchGeneration is the generation in which Coder answered a + create with a key that did not match the request. The controller + creates no key until the generation changes. 0 means no mismatch. + format: int64 + minimum: 0 + type: integer + appliedGeneration: + description: |- + AppliedGeneration is the last generation that the controller fully + applied. A spec change re-arms a stall that waits for one. + format: int64 + type: integer + conditions: + description: Conditions are Ready, Reconciling, and Stalled. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + consecutiveIssueFailures: + description: |- + ConsecutiveIssueFailures counts create requests in a row whose outcome + was uncertain, and failed Secret writes. A success resets it. + format: int32 + minimum: 0 + type: integer + currentKey: + description: CurrentKey is the key that the output Secret holds. + properties: + id: + description: ID is the Coder key ID. It is not the key. + format: uuid + maxLength: 36 + type: string + issuedAt: + description: IssuedAt is when the controller wrote the key to + the Secret. + format: date-time + type: string + name: + description: Name is the Coder key name. + maxLength: 64 + type: string + serial: + description: Serial is the key serial. + format: int64 + minimum: 0 + type: integer + required: + - id + - issuedAt + - name + - serial + type: object + killSwitchGeneration: + description: |- + KillSwitchGeneration is the generation in which the controller found + the current key deleted in Coder. The controller creates no key until + the generation changes. 0 means the kill switch is not active. + format: int64 + minimum: 0 + type: integer + lastCreateAttemptTime: + description: |- + LastCreateAttemptTime is when the controller last sent a key create + request. Orphaned keys are deleted only after a settle window from this + time. + format: date-time + type: string + lastDriftReissueTime: + description: |- + LastDriftReissueTime is when the controller last issued a key because + the Secret no longer held the current key. + format: date-time + type: string + lastHandledTrigger: + description: |- + LastHandledTrigger is the spec.rotation.trigger value that the + controller last acted on. + maxLength: 63 + type: string + nextRotationTime: + description: NextRotationTime is when the next scheduled rotation + is due. + format: date-time + type: string + nextSerial: + description: |- + NextSerial is the serial of the next key create request. It never + decreases, so the controller never reuses a key name. + format: int64 + minimum: 0 + type: integer + observedGeneration: + description: ObservedGeneration is the generation the controller last + processed. + format: int64 + type: integer + pendingKey: + description: |- + PendingKey is set before every key create request and cleared after it. + While it is set, a create request can be in flight or already committed + in Coder. + properties: + createAttemptTime: + description: CreateAttemptTime is when the controller wrote this + marker. + format: date-time + type: string + name: + description: Name is the Coder key name of the create request. + maxLength: 64 + type: string + serial: + description: Serial is the key serial of the create request. + format: int64 + minimum: 0 + type: integer + required: + - createAttemptTime + - name + - serial + type: object + retiringKeys: + description: RetiringKeys are replaced keys that wait for deletion + in Coder. + items: + description: |- + CoderAIGatewayKeyRetiring records a replaced key that waits for deletion in + Coder. It holds no secret data and nothing derived from it. + properties: + id: + description: ID is the Coder key ID. It is not the key. + format: uuid + maxLength: 36 + type: string + name: + description: Name is the Coder key name. + maxLength: 64 + type: string + replacedAt: + description: |- + ReplacedAt is when a newer key replaced this key in the Secret. The + grace period counts from this time. + format: date-time + type: string + serial: + description: Serial is the key serial. + format: int64 + minimum: 0 + type: integer + required: + - id + - name + - replacedAt + - serial + type: object + maxItems: 3 + type: array + x-kubernetes-list-map-keys: + - serial + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 @@ -1229,8 +1616,7 @@ spec: aiGateway: description: |- AIGateway configures CoderAIGatewayKey objects that use this control - plane. It has no effect yet: the CoderAIGatewayKey controller is not - shipped. + plane. properties: allowKeys: description: |- @@ -1242,9 +1628,6 @@ spec: controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. - - This field has no effect yet: the CoderAIGatewayKey controller is not - shipped. type: boolean type: object certs: @@ -6107,12 +6490,6 @@ rules: - coder.com resources: - coderaigatewaykeys - - codermcpservers - verbs: - - list -- apiGroups: - - coder.com - resources: - coderaiproviders verbs: - get @@ -6122,6 +6499,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/finalizers - coderaiproviders/finalizers - codercontrolplanes/finalizers - coderprovisioners/finalizers @@ -6132,6 +6510,7 @@ rules: - apiGroups: - coder.com resources: + - coderaigatewaykeys/status - coderaiproviders/status verbs: - get @@ -6161,6 +6540,12 @@ rules: - get - patch - update +- apiGroups: + - coder.com + resources: + - codermcpservers + verbs: + - list - apiGroups: - coder.com resources: diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 75a4d256c..c6a02ff4f 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -31,7 +31,7 @@ graph TD | --- | --- | --- | --- | | Code | `internal/app/controllerapp/`, `internal/controller/` | `internal/app/apiserverapp/`, `internal/aggregated/storage/`, `internal/aggregated/coder/` | `internal/app/mcpapp/` | | Listens on | `:8081` (`/healthz`, `/readyz`) | `:6443` HTTPS (default) | `127.0.0.1:8090` only. `/mcp` requires the bearer token. `/healthz` and `/readyz` do not. | -| Resources | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider` | `coderworkspaces`, `codertemplates`, `codertemplateversions` | Tools for control planes, templates, workspaces, events, pod logs, and run state | +| Resources | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, `CoderAIGatewayKey` | `coderworkspaces`, `codertemplates`, `codertemplateversions` | Tools for control planes, templates, workspaces, events, pod logs, and run state | ### Controller @@ -75,7 +75,7 @@ graph TD | Path | Contents | | --- | --- | -| `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, and `CoderAIProvider` | +| `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, and `CoderAIGatewayKey` | | `config/rbac/` | The `coder-k8s` ServiceAccount, the `manager-role` ClusterRole, and the bindings (including auth-delegator) | | `config/apiserver-standalone/` | For standalone `--app=aggregated-apiserver`: the `coder-k8s-apiserver` ServiceAccount, its RBAC, and the placeholder Secret for the serving CA | | `deploy/deployment.yaml` | The `coder-k8s` Deployment (default `--app=all`) | diff --git a/docs/how-to/deploy-controller.md b/docs/how-to/deploy-controller.md index d83ac64f3..ee0691edd 100644 --- a/docs/how-to/deploy-controller.md +++ b/docs/how-to/deploy-controller.md @@ -1,6 +1,6 @@ # Deploy the controller -Run `coder-k8s` as an operator only (`--app=controller`). The operator reconciles `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, and `CoderAIProvider` resources. +Run `coder-k8s` as an operator only (`--app=controller`). The operator reconciles `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, and `CoderAIGatewayKey` resources. Run the commands from a clone of this repository. diff --git a/docs/how-to/manage-ai-gateway-keys.md b/docs/how-to/manage-ai-gateway-keys.md new file mode 100644 index 000000000..80a03755e --- /dev/null +++ b/docs/how-to/manage-ai-gateway-keys.md @@ -0,0 +1,206 @@ +# Manage AI Gateway keys with CoderAIGatewayKey + +A `CoderAIGatewayKey` keeps one Coder AI Gateway key in a Secret. The operator creates the key in Coder (Coder's `/api/v2/ai-gateway/keys`), writes it into a Secret that the object owns, rotates it, deletes replaced keys after a grace period, and deletes every key of the object when you delete the object. A standalone AI Gateway reads the key from the Secret. + +!!! warning "A gateway key is a full-trust credential" + Coder trusts every holder of a gateway key. The holder can load the API keys of every enabled AI provider, read the external-auth tokens that users gave to MCP servers, and act as any user on the gateway. Treat the output Secret like the operator token Secret. Grant `create`, `update`, `patch` and `delete` on `coderaigatewaykeys`, and `get` on the output Secrets, only to principals who may hold that power. + +## Before you start + +- A Coder Premium license with the AI Gateway enabled (`CODER_AI_GATEWAY_ENABLED`). Without it, Coder answers `403` and the object stalls with `LicenseRequired`. +- A `CoderControlPlane` in the same namespace with `status.operatorAccessReady: true`. The operator user must be a site owner. +- Node clocks within 2 minutes of each other, and the API server's default request timeout of 60 seconds (`--request-timeout`). The operator waits a 5-minute settle window after each key create before it treats a key as an orphan. That window holds only under these two conditions. With a longer request timeout, a late create can commit after the window, and its key then stays in Coder. + +## 1. Allow keys on the control plane + +The control plane admin opts in. Without this, the operator creates no key and the object stalls with `NotAllowed`. An object that never created a key makes no Coder call. + +```bash +kubectl -n coder patch codercontrolplane coder --type=merge -p '{"spec":{"aiGateway":{"allowKeys":true}}}' +``` + +This delegates the power of a gateway key to everyone who can create `CoderAIGatewayKey` objects in the namespace. Setting `allowKeys` back to `false` stops new keys and rotations. The current key of each object stays in its Secret and in Coder until you delete the object. + +## 2. Create the key + +```yaml +apiVersion: coder.com/v1alpha1 +kind: CoderAIGatewayKey +metadata: + name: gateway + namespace: coder +spec: + controlPlaneRef: + name: coder + secretName: coder-ai-gateway-key + rotation: + intervalSeconds: 2592000 # 30 days (default). 0 turns scheduled rotation off. + gracePeriodSeconds: 3600 # 1 hour (default), between 10 minutes and 7 days. + consumerWaitSeconds: 86400 # 1 day (default), at most 7 days. +``` + +```bash +kubectl apply -f gateway-key.yaml +kubectl -n coder wait --for=condition=Ready coderaigatewaykey/gateway --timeout=2m +``` + +The sample is in `config/samples/coder_v1alpha1_coderaigatewaykey.yaml`. `spec.controlPlaneRef` and `spec.secretName` cannot change. To move the key to another Secret, delete the object and create a new one. + +The operator creates the Secret and owns it. It holds one data key, `key`, which is the default of the AI Gateway Helm chart. The annotations `coder.com/ai-gateway-key-id` and `coder.com/ai-gateway-key-name` name the current key and change on each rotation. The operator keeps every other label and annotation, for example reloader annotations. + +The operator never writes a Secret that it does not own. An existing Secret with that name stalls the object with `SecretNotOwned`, and the Secret stays unchanged. Names that a control plane or another Coder resource in the namespace uses stall with `SecretNameReserved`. + +The key names in Coder are `ck8s--`, where `` is the object's `metadata.uid` without dashes. Status holds key IDs, names and times only, never the key or its prefix. + +## 3. Use the key in a gateway + +The gateway reads the key once, at startup. A new key in the Secret takes effect only when the gateway pods restart. Set these in the AI Gateway Helm chart: + +- `aigateway.keySecret.name` to `spec.secretName`. +- A reloader, such as Stakater Reloader, that restarts the pods when the Secret changes, for example the pod annotation `secret.reloader.stakater.com/reload: coder-ai-gateway-key`. +- At least 2 replicas, so a restart causes no gap. + +Without a reloader, run `kubectl rollout restart` on the gateway Deployment after each rotation. Without either, the operator deletes the old key when `consumerWaitSeconds` ends. Each old pod then fails its next connection, exits and restarts with the new key, so each replica has a short outage. + +## 4. Read the result + +```bash +kubectl -n coder get coderaigatewaykeys +kubectl -n coder get coderaigatewaykey gateway -o jsonpath='{range .status.conditions[*]}{.type}={.status} {.reason}: {.message}{"\n"}{end}' +``` + +The object has the conditions `Ready`, `Reconciling` and `Stalled`. `Ready=True` means that the Secret holds the current key and Coder lists it. A rotation keeps `Ready=True`, because the Secret stays valid all the time. A cleanup failure next to a valid Secret shows `Ready=True` together with `Stalled=True`, and its message starts with "The key in Secret ... still works." + +| Reason | Meaning | +| --- | --- | +| `Ready` | The Secret holds the current key, and Coder lists it. | +| `IssuingKey` | The operator creates a key, or retries a create whose answer was uncertain. | +| `WaitingForGracePeriod`, `WaitingForConsumers`, `RetiringKey` | A replaced key waits for deletion. See [Rotation](#rotation). | +| `RotationBlocked` | A rotation is due, but replaced keys still wait for deletion. | +| `DeletingOrphans` | A key of this object that no status names waits for the settle window, then the operator deletes it. | +| `ControlPlaneNotReady`, `OperatorAccessNotReady` | The control plane cannot serve Coder calls yet. | +| `CoderUnavailable` | A Coder call failed. The operator retries every 30 seconds. | +| `NotAllowed` | The control plane does not allow keys. See [step 1](#1-allow-keys-on-the-control-plane). | +| `LicenseRequired`, `CoderRejected`, `NotSupported` | Coder refused: no Premium license or AI Gateway, an operator user that is not a site owner, or a Coder version without the key API. | +| `SecretNotOwned`, `SecretNameReserved`, `SecretWriteRejected` | The output Secret cannot be used. Choose another `secretName` (delete and re-create the object), or fix the admission policy that rejects the write. | +| `KeyDeletedExternally` | Someone deleted the current key in Coder. See [Kill switch](#kill-switch). | +| `SecretConflict` | The Secret lost the key again soon after a re-issue. See [Secret drift](#secret-drift). | +| `IssueFailing`, `CoderAnswerMismatch` | Key creates failed 10 times in a row, or Coder answered a create with a key that does not match the request. See [Stalls](#stalls). | +| `DeletingKeys` | The object is being deleted. See [Delete a key](#delete-a-key). | + +## Rotation + +A scheduled rotation is due `intervalSeconds` after the current key was written (`status.nextRotationTime`). A requested rotation starts each time `spec.rotation.trigger` changes to a new non-empty value: + +```bash +kubectl -n coder patch coderaigatewaykey gateway --type=merge -p '{"spec":{"rotation":{"trigger":"2026-10-14"}}}' +``` + +A rotation creates a new key, writes it into the Secret, and keeps the replaced key in `status.retiringKeys`. The operator deletes a replaced key after `gracePeriodSeconds`, once Coder shows that no gateway uses it: it has no heartbeat for 10 minutes, or the new key's heartbeat is newer. While a gateway still uses the old key, the object shows `WaitingForConsumers`: restart the gateway pods. The operator deletes the old key when `consumerWaitSeconds` ends anyway. + +At most 3 replaced keys wait at a time. A scheduled rotation starts only while no key waits, and a requested rotation only while fewer than 2 wait. Otherwise the object shows `RotationBlocked` and the rotation starts when a slot frees. + +## Secret drift + +When the Secret loses the current key (someone edits or deletes it), the operator creates a new key, writes it, and lets the old key retire with the full grace period. When the Secret loses the key again within the grace period, the object stalls with `SecretConflict`: another writer, such as a GitOps tool, likely manages the same Secret. Remove the other writer. The stall ends when the grace period after the last re-issue ends. With 3 replaced keys waiting, it ends when the operator deletes one of them. + +## Kill switch + +Delete the current key in Coder to revoke it at once. Gateways that use it lose their connection within 60 seconds. + +```bash +coder ai-gateway keys list +coder ai-gateway keys delete +``` + +The object then stalls with `KeyDeletedExternally`, and the operator creates no key until the spec changes. Edit the spec, for example change `spec.rotation.trigger`, to issue a new key. Two cases need a second spec edit: a spec change made before the operator saw the deleted key, and a spec edit that ended a `SecretNotOwned`, `SecretNameReserved` or `NotAllowed` stall while the key was already deleted. + +## Stalls + +A stalled object creates no key until its spec changes. It still deletes orphans and replaced keys. + +- `IssueFailing`: 10 key creates in a row had an uncertain answer, or their Secret write failed. Fix the cause in the message, then edit the spec. +- `CoderAnswerMismatch`: Coder answered a create with a key that does not match the request. The created key is an orphan, and the operator deletes it after the settle window. Check the Coder version and any proxy between the operator and Coder, then edit the spec. + +## Delete a key + +Deleting the object deletes every key of the object in Coder, the current key included. Gateways that use them lose access within 60 seconds. The Secret goes with the object. + +1. The operator waits until 5 minutes after its last key create, so that a create still in flight commits first (`DeletingKeys`). +2. It lists the keys, deletes each key of this object, and removes the finalizer `coder.com/ai-gateway-key-cleanup` only after a list shows none. + +The finalizer stays while Coder cannot confirm the deletion: + +| Condition | Retry | +| --- | --- | +| `CoderUnavailable`: Coder does not answer, answers `5xx` or `429`, or answers `404` that is not Coder's own (for example an HTML page from a proxy) | Every 60 seconds, with no deadline | +| `ControlPlaneNotReady`, `OperatorAccessNotReady` | Every 60 seconds | +| `LicenseRequired`, `CoderRejected`, `NotSupported` (`Stalled=True`) | Every 10 minutes | + +When the control plane is gone, nothing can delete the keys. The finalizer goes at once. The operator logs `Releasing the finalizer without deleting the keys in Coder` with reason `ControlPlaneGone` and the key IDs: delete those keys with `coder ai-gateway keys delete`. + +While the control plane is being deleted and its operator access still works, the operator runs one pass of at most 10 seconds without the settle window, then removes the finalizer whatever the answer. A key create that was still in flight at that moment can commit later, and its key stays in Coder. When the pass fails, the log lists the IDs that it did not delete. + +!!! warning "Delete the keys before the namespace" + When you delete the namespace, Kubernetes deletes the control plane and the `CoderAIGatewayKey` objects at the same time. The control plane is usually gone before the operator handles the objects, so the finalizers go without a delete, and the keys stay valid in Coder. The namespace deletion finishes, and the operator logs the key IDs with reason `ControlPlaneGone`. + +Delete the `CoderAIGatewayKey` objects first and wait until they are gone, then delete the control plane or the namespace: + +```bash +kubectl -n coder delete coderaigatewaykeys --all --wait --timeout=420s +``` + +If the namespace is already gone, delete the logged keys with `coder ai-gateway keys delete`. + +### Stuck finalizer + +A deletion that keeps failing never ends by itself. Leave it in one of these ways: + +1. **Fix Coder or the operator user.** Restore Coder access, the Premium license or the AI Gateway, the operator user's site-owner role, or the proxy that answers `404`. The next retry deletes the keys and releases the object. +2. **Delete the control plane.** The operator then runs its one bounded pass and releases the object. Do this only when you remove that Coder deployment anyway. +3. **Remove the finalizer by hand, then delete the keys in Coder.** + + Note the key names from `status.currentKey` and `status.retiringKeys`, and the object's UID (`kubectl -n coder get coderaigatewaykey gateway -o jsonpath='{.metadata.uid}'`). Find the index of the finalizer: + + ```bash + kubectl -n coder get coderaigatewaykey gateway -o jsonpath='{.metadata.finalizers}{"\n"}' + ``` + + Remove it with a JSON patch. The `test` operation makes the patch fail, with no change, if another finalizer is at that index. Replace `0` with the index: + + ```bash + kubectl -n coder patch coderaigatewaykey gateway --type=json -p '[ + {"op": "test", "path": "/metadata/finalizers/0", "value": "coder.com/ai-gateway-key-cleanup"}, + {"op": "remove", "path": "/metadata/finalizers/0"} + ]' + ``` + + Then delete every key whose name starts with `ck8s--` with `coder ai-gateway keys delete `. + +## Known limits + +- **Leftover keys.** A key that commits more than 5 minutes after a lost create answer, or after the object is gone, stays in Coder. So do the keys of objects that go together with their control plane, for example in a namespace deletion. Find keys of deleted objects with `coder ai-gateway keys list`: their names start with `ck8s-` and a UID that no `CoderAIGatewayKey` has. +- **A short key list.** The operator reads all keys with one list call. When a list answer names the current key but leaves out a replaced key, the operator treats the replaced key as deleted. If the key appears again, the operator deletes it as an orphan, without the rest of its grace period. Coder lists all keys in one answer, so only a proxy between the operator and Coder that changes the answer can cause this. + +## Grant access + +| Permission | Coder power | +| --- | --- | +| `create`, `update`, `patch` on `coderaigatewaykeys` (with `allowKeys: true`) | Mint full-trust gateway keys with the operator token | +| `delete` on `coderaigatewaykeys` | Revokes the object's keys for every gateway that uses them | +| `get` on the output Secret | Holds a full-trust gateway key | +| `get`, `list` on `coderaigatewaykeys` | Sees Secret names, key names, Coder key IDs and times. No secrets. | + +A viewer Role: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: coderaigatewaykey-viewer + namespace: coder +rules: + - apiGroups: ["coder.com"] + resources: ["coderaigatewaykeys"] + verbs: ["get", "list", "watch"] +``` diff --git a/docs/index.md b/docs/index.md index c82d1a7e7..9016006f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ Use native Kubernetes APIs to run and manage [Coder](https://coder.com). | Component | Resources | | --- | --- | -| Operator | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider` (`coder.com/v1alpha1`) | +| Operator | `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, `CoderAIGatewayKey` (`coder.com/v1alpha1`) | | Aggregated API server | `CoderWorkspace`, `CoderTemplate`, `CoderTemplateVersion` (`aggregation.coder.com/v1alpha1`) | | MCP server | Operational tools over HTTP | diff --git a/docs/reference/api/coderaigatewaykey.md b/docs/reference/api/coderaigatewaykey.md new file mode 100644 index 000000000..8d16fb076 --- /dev/null +++ b/docs/reference/api/coderaigatewaykey.md @@ -0,0 +1,99 @@ + + +# `CoderAIGatewayKey` + +## API identity + +- Group/version: `coder.com/v1alpha1` +- Kind: `CoderAIGatewayKey` +- Resource: `coderaigatewaykeys` +- Scope: namespaced + +## Spec + +| Field | Type | Description | +| --- | --- | --- | +| `controlPlaneRef` | [CoderControlPlaneReference](#codercontrolplanereference) | ControlPlaneRef names the CoderControlPlane in the same namespace. The controller creates and deletes keys with that control plane's operator token. The control plane must allow keys in spec.aiGateway.allowKeys. | +| `secretName` | string | SecretName is the Secret in the same namespace that receives the key, under the data key "key". The controller creates the Secret and owns it. It never writes a Secret that already exists and that this object does not own. To move the key to another Secret, delete and re-create this object. | +| `rotation` | [CoderAIGatewayKeyRotationSpec](#coderaigatewaykeyrotationspec) | Rotation configures when the controller replaces the key and how long a replaced key stays valid. | + +## Status + +| Field | Type | Description | +| --- | --- | --- | +| `observedGeneration` | integer | ObservedGeneration is the generation the controller last processed. | +| `appliedGeneration` | integer | AppliedGeneration is the last generation that the controller fully applied. A spec change re-arms a stall that waits for one. | +| `nextSerial` | integer | NextSerial is the serial of the next key create request. It never decreases, so the controller never reuses a key name. | +| `lastCreateAttemptTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | LastCreateAttemptTime is when the controller last sent a key create request. Orphaned keys are deleted only after a settle window from this time. | +| `pendingKey` | [CoderAIGatewayKeyPending](#coderaigatewaykeypending) | PendingKey is set before every key create request and cleared after it. While it is set, a create request can be in flight or already committed in Coder. | +| `currentKey` | [CoderAIGatewayKeyCurrent](#coderaigatewaykeycurrent) | CurrentKey is the key that the output Secret holds. | +| `retiringKeys` | [CoderAIGatewayKeyRetiring](#coderaigatewaykeyretiring) array | RetiringKeys are replaced keys that wait for deletion in Coder. | +| `lastHandledTrigger` | string | LastHandledTrigger is the spec.rotation.trigger value that the controller last acted on. | +| `nextRotationTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | NextRotationTime is when the next scheduled rotation is due. | +| `consecutiveIssueFailures` | integer | ConsecutiveIssueFailures counts create requests in a row whose outcome was uncertain, and failed Secret writes. A success resets it. | +| `lastDriftReissueTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | LastDriftReissueTime is when the controller last issued a key because the Secret no longer held the current key. | +| `killSwitchGeneration` | integer | KillSwitchGeneration is the generation in which the controller found the current key deleted in Coder. The controller creates no key until the generation changes. 0 means the kill switch is not active. | +| `answerMismatchGeneration` | integer | AnswerMismatchGeneration is the generation in which Coder answered a create with a key that did not match the request. The controller creates no key until the generation changes. 0 means no mismatch. | +| `conditions` | [Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array | Conditions are Ready, Reconciling, and Stalled. | + +## Referenced types + +### CoderAIGatewayKeyCurrent + +CoderAIGatewayKeyCurrent records the key that the output Secret holds. It +holds no secret data and nothing derived from it. + +| Field | Type | Description | +| --- | --- | --- | +| `serial` | integer | Serial is the key serial. | +| `id` | string | ID is the Coder key ID. It is not the key. | +| `name` | string | Name is the Coder key name. | +| `issuedAt` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | IssuedAt is when the controller wrote the key to the Secret. | + +### CoderAIGatewayKeyPending + +CoderAIGatewayKeyPending is the create marker. The controller writes it +before every key create request. It holds no secret data. + +| Field | Type | Description | +| --- | --- | --- | +| `serial` | integer | Serial is the key serial of the create request. | +| `name` | string | Name is the Coder key name of the create request. | +| `createAttemptTime` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | CreateAttemptTime is when the controller wrote this marker. | + +### CoderAIGatewayKeyRetiring + +CoderAIGatewayKeyRetiring records a replaced key that waits for deletion in +Coder. It holds no secret data and nothing derived from it. + +| Field | Type | Description | +| --- | --- | --- | +| `serial` | integer | Serial is the key serial. | +| `id` | string | ID is the Coder key ID. It is not the key. | +| `name` | string | Name is the Coder key name. | +| `replacedAt` | [Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta) | ReplacedAt is when a newer key replaced this key in the Secret. The grace period counts from this time. | + +### CoderAIGatewayKeyRotationSpec + +CoderAIGatewayKeyRotationSpec configures key rotation. ++kubebuilder:validation:XValidation:rule="!has(self.intervalSeconds) || !has(self.gracePeriodSeconds) || self.intervalSeconds == 0 || self.gracePeriodSeconds < self.intervalSeconds",message="gracePeriodSeconds must be shorter than intervalSeconds" + +| Field | Type | Description | +| --- | --- | --- | +| `intervalSeconds` | integer | IntervalSeconds is the time between scheduled rotations, counted from the time the current key was written to the Secret. 0 turns scheduled rotation off. Otherwise it is between 1 day and 365 days. The default is 30 days, because Coder keys never expire. | +| `gracePeriodSeconds` | integer | GracePeriodSeconds is the minimum time between writing a new key to the Secret and deleting the replaced key in Coder. Consumers must reload within it. Between 10 minutes and 7 days, default 1 hour. | +| `consumerWaitSeconds` | integer | ConsumerWaitSeconds is how long after the grace period the controller keeps a replaced key while Coder still reports connections that use it. After that time the controller deletes the key, and gateways that still use it exit and restart with the key from the Secret. 0 deletes the key when the grace period ends. At most 7 days, default 1 day. | +| `trigger` | string | Trigger starts one rotation each time it changes to a new non-empty value. The value set when the object is created starts no rotation. | + +### CoderControlPlaneReference + +CoderControlPlaneReference names a CoderControlPlane in the same namespace. + +| Field | Type | Description | +| --- | --- | --- | +| `name` | string | Name is the CoderControlPlane name. | + +## Source + +- Go type: `api/v1alpha1/coderaigatewaykey_types.go` +- Generated CRD: `config/crd/bases/coder.com_coderaigatewaykeys.yaml` diff --git a/docs/reference/api/codercontrolplane.md b/docs/reference/api/codercontrolplane.md index ab5aecd7b..1a5da6f3e 100644 --- a/docs/reference/api/codercontrolplane.md +++ b/docs/reference/api/codercontrolplane.md @@ -41,7 +41,7 @@ | `affinity` | [Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#affinity-v1-core) | Affinity configures pod affinity/anti-affinity rules. | | `topologySpreadConstraints` | [TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#topologyspreadconstraint-v1-core) array | TopologySpreadConstraints control pod topology spread. | | `mcpServers` | [MCPServersSpec](#mcpserversspec) | MCPServers configures CoderMCPServer objects that use this control plane. It has no effect yet: the CoderMCPServer controller is not shipped. | -| `aiGateway` | [AIGatewaySpec](#aigatewayspec) | AIGateway configures CoderAIGatewayKey objects that use this control plane. It has no effect yet: the CoderAIGatewayKey controller is not shipped. | +| `aiGateway` | [AIGatewaySpec](#aigatewayspec) | AIGateway configures CoderAIGatewayKey objects that use this control plane. | | `workspacePools` | [WorkspacePoolsSpec](#workspacepoolsspec) | WorkspacePools configures CoderWorkspacePool objects that use this control plane. It has no effect yet: the CoderWorkspacePool controller is not shipped. | | `templateTests` | [TemplateTestsSpec](#templatetestsspec) | TemplateTests configures CoderTemplateTest runs against this control plane. | @@ -71,7 +71,7 @@ CoderControlPlane (CoderControlPlane.spec.aiGateway). | Field | Type | Description | | --- | --- | --- | -| `allowKeys` | boolean | AllowKeys lets CoderAIGatewayKey objects in this namespace create and rotate Coder AI Gateway keys with this control plane's operator token. A gateway key lets its holder load the API key of every enabled AI provider, so this delegates that power to everyone who can create CoderAIGatewayKey objects here. When false (the default), the controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. This field has no effect yet: the CoderAIGatewayKey controller is not shipped. | +| `allowKeys` | boolean | AllowKeys lets CoderAIGatewayKey objects in this namespace create and rotate Coder AI Gateway keys with this control plane's operator token. A gateway key lets its holder load the API key of every enabled AI provider, so this delegates that power to everyone who can create CoderAIGatewayKey objects here. When false (the default), the controller creates no key. Setting it back to false stops new keys and rotations. The current key of each CoderAIGatewayKey stays until that object is deleted. | ### CertSecretSelector diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md index 5d76621c5..b24c9c189 100644 --- a/docs/tutorials/getting-started.md +++ b/docs/tutorials/getting-started.md @@ -63,15 +63,16 @@ Then open `http://127.0.0.1:3000` in a browser. ## 5. Clean up (optional) -Delete the `CoderAIProvider` objects first, if you created any, and then the control plane, while the operator still runs. The operator then deletes the providers in Coder, cleans up, and removes its finalizers. A provider deleted after its control plane stays in Coder. After that, remove the bundle. The commands set `BASE` again, because you can be in a new shell. +Delete the `CoderAIProvider` and `CoderAIGatewayKey` objects first, if you created any, and then the control plane, while the operator still runs. The operator then deletes the providers and keys in Coder, cleans up, and removes its finalizers. A provider or key deleted after its control plane stays in Coder. A `CoderAIGatewayKey` can take up to 5 minutes after its last key create. After that, remove the bundle. The commands set `BASE` again, because you can be in a new shell. !!! warning - Delete all other `CoderAIProvider`, `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` resources before you delete the bundle. Delete `CoderAIProvider` resources before their control plane. When you delete the bundle, you also delete the `coder.com` CRDs, and Kubernetes then removes all remaining resources of these kinds in the cluster. When you delete the `coder` namespace, you also delete all other objects in it, for example Secrets and PersistentVolumeClaims. + Delete all other `CoderAIGatewayKey`, `CoderAIProvider`, `CoderControlPlane`, `CoderProvisioner`, and `CoderWorkspaceProxy` resources before you delete the bundle. Delete `CoderAIProvider` and `CoderAIGatewayKey` resources before their control plane. When you delete the bundle, you also delete the `coder.com` CRDs, and Kubernetes then removes all remaining resources of these kinds in the cluster. When you delete the `coder` namespace, you also delete all other objects in it, for example Secrets and PersistentVolumeClaims. ```bash BASE="https://raw.githubusercontent.com/coder/coder-k8s/main" kubectl delete coderaiproviders --all -n coder --wait --timeout=120s +kubectl delete coderaigatewaykeys --all -n coder --wait --timeout=420s kubectl delete -f "$BASE/config/samples/coder_v1alpha1_codercontrolplane.yaml" --ignore-not-found kubectl wait --for=delete codercontrolplane/codercontrolplane-sample -n coder --timeout=120s kubectl delete -f "$BASE/dist/install.yaml" --ignore-not-found diff --git a/hack/update-manifests.sh b/hack/update-manifests.sh index a04309fc8..841e7a4f5 100755 --- a/hack/update-manifests.sh +++ b/hack/update-manifests.sh @@ -31,7 +31,7 @@ list_manifests() { # CRDs of kinds whose controller is not registered yet. They go to config/crd/dormant/, which no kustomization, # example, doc or script applies, so the kind is never served without its controller. # Move a file back by deleting its entry when the controller ships. -DORMANT_CRDS=(coder.com_coderaigatewaykeys.yaml coder.com_codermcpservers.yaml coder.com_coderworkspacepools.yaml coder.com_coderworkspacepoolmembers.yaml coder.com_coderworkspaceclaims.yaml) +DORMANT_CRDS=(coder.com_codermcpservers.yaml coder.com_coderworkspacepools.yaml coder.com_coderworkspacepoolmembers.yaml coder.com_coderworkspaceclaims.yaml) # RBAC manifests that only the aggregated API server needs; the controller-only install bundle leaves them out. BUNDLE_EXCLUDED_RBAC=(apiservice-cabundle-role.yaml auth-delegator-binding.yaml authentication-reader-binding.yaml) diff --git a/internal/app/controllerapp/controllerapp.go b/internal/app/controllerapp/controllerapp.go index 360be7674..d0ca9b118 100644 --- a/internal/app/controllerapp/controllerapp.go +++ b/internal/app/controllerapp/controllerapp.go @@ -134,6 +134,16 @@ func SetupControllers(mgr manager.Manager) error { return fmt.Errorf("unable to create AI provider controller: %w", err) } + aiGatewayKeyReconciler := &controller.CoderAIGatewayKeyReconciler{ + Client: client, + APIReader: mgr.GetAPIReader(), + Scheme: managerScheme, + Clock: clock.RealClock{}, + } + if err := aiGatewayKeyReconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create AI Gateway key controller: %w", err) + } + return nil } diff --git a/internal/controller/coderaigatewaykey_controller.go b/internal/controller/coderaigatewaykey_controller.go index 2c91578e1..8c5b12e73 100644 --- a/internal/controller/coderaigatewaykey_controller.go +++ b/internal/controller/coderaigatewaykey_controller.go @@ -69,10 +69,13 @@ const ( aiGatewayKeyNameAnnotation = "coder.com/ai-gateway-key-name" ) +// +kubebuilder:rbac:groups=coder.com,resources=coderaigatewaykeys,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=coder.com,resources=coderaigatewaykeys/status,verbs=get;update +// +kubebuilder:rbac:groups=coder.com,resources=coderaigatewaykeys/finalizers,verbs=update + // CoderAIGatewayKeyReconciler reconciles CoderAIGatewayKey objects (#156). It -// is not registered yet. This version runs the checks of the observe step -// (plan 4.2) and claims the output Secret. It creates and deletes no Coder -// key. +// issues a Coder AI Gateway key into an owned Secret, rotates it, retires +// replaced keys, and deletes every key of the object when the object goes. type CoderAIGatewayKeyReconciler struct { client.Client // APIReader reads the output Secret and the objects of the reserved-name @@ -105,10 +108,9 @@ func aiGatewayKeyStalled(reason, format string, args ...any) *aiGatewayKeyStep { return &aiGatewayKeyStep{stalled: true, reason: reason, message: fmt.Sprintf(format, args...), requeue: aiGatewayKeyResync} } -// SetupWithManager wires the reconciler into controller-runtime. Nothing -// calls it until activation (K10). A change of the output Secret, or of any -// control plane in the namespace (any of them can reserve the Secret name), -// requeues the object at once. +// SetupWithManager wires the reconciler into controller-runtime. A change of +// the output Secret, or of any control plane in the namespace (any of them +// can reserve the Secret name), requeues the object at once. func (r *CoderAIGatewayKeyReconciler) SetupWithManager(mgr ctrl.Manager) error { if mgr == nil || r.Client == nil || r.APIReader == nil || r.Scheme == nil || r.Clock == nil { return errors.New("assertion failed: AI Gateway key reconciler needs a manager, a client, an API reader, a scheme, and a clock") diff --git a/internal/controller/coderaigatewaykey_controller_test.go b/internal/controller/coderaigatewaykey_controller_test.go index 10d66a4ee..2ec30dd04 100644 --- a/internal/controller/coderaigatewaykey_controller_test.go +++ b/internal/controller/coderaigatewaykey_controller_test.go @@ -374,6 +374,46 @@ func requireAIGatewayKeyReady(t *testing.T, key *coderv1alpha1.CoderAIGatewayKey } } +// The allowKeys gate on a live object (decision D2): without the block or +// with false, the controller creates no key (NotAllowed, zero Coder calls). +// True issues a key. False again stops new keys and rotations, and the +// current key stays in the Secret and in Coder. +func TestCoderAIGatewayKeyAllowKeysGate(t *testing.T) { + t.Parallel() + ctx := context.Background() + env := newAIGatewayKeyEnv(ctx, t, nil) + got, _ := env.reconcile(ctx, t, env.createKey(ctx, t, "coder", testAIGatewaySecret)) + requireAIGatewayKeyStep(t, got, true, "NotAllowed") + setAllowKeys := func(allow bool) { + cp := &coderv1alpha1.CoderControlPlane{} + require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Namespace: env.namespace, Name: "coder"}, cp)) + cp.Spec.AIGateway = &coderv1alpha1.AIGatewaySpec{AllowKeys: allow} + require.NoError(t, k8sClient.Update(ctx, cp)) + } + setAllowKeys(false) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, true, "NotAllowed") + require.Zero(t, env.fake.totalRequests()) + requireNoOutputSecret(ctx, t, env.namespace, testAIGatewaySecret) + + setAllowKeys(true) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyReady(t, got) + current := got.Status.CurrentKey + + setAllowKeys(false) + got.Spec.Rotation.Trigger = "rotate-1" + require.NoError(t, k8sClient.Update(ctx, got)) + got, _ = env.reconcile(ctx, t, got) + requireAIGatewayKeyStep(t, got, true, "NotAllowed") + require.Equal(t, current.ID, got.Status.CurrentKey.ID, "no rotation") + require.Equal(t, 1, env.fake.requestCount(routeCreateAIGatewayKey)) + require.Len(t, env.fake.aiGatewayKeys(), 1, "the current key stays in Coder") + secret := &corev1.Secret{} + require.NoError(t, k8sClient.Get(ctx, client.ObjectKey{Namespace: env.namespace, Name: testAIGatewaySecret}, secret)) + require.Equal(t, env.fake.aiGatewayKeys()[0].plaintext, string(secret.Data[coderv1alpha1.CoderAIGatewayKeySecretDataKey]), "the Secret keeps the key") +} + // The object that claimed a shared Secret keeps it. An object created later // in the same second with a lower name ranks older, but it must not stall // the claimant: it reports SecretNotOwned, and the claimant goes on. The diff --git a/internal/controller/codercontrolplane_aigateway_api_test.go b/internal/controller/codercontrolplane_aigateway_api_test.go index 82a09c740..2fb223acf 100644 --- a/internal/controller/codercontrolplane_aigateway_api_test.go +++ b/internal/controller/codercontrolplane_aigateway_api_test.go @@ -47,8 +47,8 @@ func TestControlPlaneAIGatewayAllowKeysRoundTrip(t *testing.T) { } } -// The field has no effect until the CoderAIGatewayKey controller ships: -// turning it on must not roll the Coder Deployment. +// The field gates only the CoderAIGatewayKey controller: turning it on must +// not roll the Coder Deployment. func TestControlPlaneAIGatewayAllowKeysLeavesDeploymentUnchanged(t *testing.T) { ensureGatewaySchemeRegistered(t) ctx := context.Background() diff --git a/internal/controller/controllerapp_setup_test.go b/internal/controller/controllerapp_setup_test.go new file mode 100644 index 000000000..f9144ba7c --- /dev/null +++ b/internal/controller/controllerapp_setup_test.go @@ -0,0 +1,46 @@ +package controller_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/config" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/coder/coder-k8s/internal/app/controllerapp" +) + +// A manager with every controller of the controller app registered starts +// against the envtest API server: no field index is registered twice, every +// watched kind is served, and the caches sync. The test runs serially, and +// its cache sees only an empty namespace, so its reconcilers never touch +// the objects of other tests. +func TestSetupControllersStartsEveryController(t *testing.T) { + ctx := context.Background() + namespace := createTestNamespace(ctx, t, "setup") + scheme := controllerapp.NewScheme() + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{namespace: {}}}, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "0", + // Other tests build managers with the same controller names, and + // controller-runtime checks names across the process. + Controller: config.Controller{SkipNameValidation: ptrTo(true)}, + }) + require.NoError(t, err) + require.NoError(t, controllerapp.SetupControllers(mgr)) + + ctx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- mgr.Start(ctx) }() + syncCtx, syncCancel := context.WithTimeout(ctx, 30*time.Second) + defer syncCancel() + require.True(t, mgr.GetCache().WaitForCacheSync(syncCtx), "the caches sync") + cancel() + require.NoError(t, <-done) +} diff --git a/mkdocs.yml b/mkdocs.yml index 8e80530ae..62e3d4bd8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - Start, stop, and read logs of workspaces: how-to/start-stop-workspaces.md - Test a template version: how-to/test-templates.md - Manage AI providers: how-to/manage-ai-providers.md + - Manage AI Gateway keys: how-to/manage-ai-gateway-keys.md - Gate template promotion with GitOps: how-to/gitops.md - Run the MCP server: how-to/mcp-server.md - Troubleshooting: how-to/troubleshooting.md @@ -51,6 +52,7 @@ nav: - Aggregated API behavior: reference/aggregated-api-behavior.md - API: # BEGIN GENERATED API NAV + - CoderAIGatewayKey: reference/api/coderaigatewaykey.md - CoderAIProvider: reference/api/coderaiprovider.md - CoderControlPlane: reference/api/codercontrolplane.md - CoderProvisioner: reference/api/coderprovisioner.md