diff --git a/docs/how-to/manage-ai-providers.md b/docs/how-to/manage-ai-providers.md index b06853140..1b24a5dd9 100644 --- a/docs/how-to/manage-ai-providers.md +++ b/docs/how-to/manage-ai-providers.md @@ -9,6 +9,8 @@ A `CoderAIProvider` manages one AI provider in Coder (Coder's `/api/v2/ai/provid - A `CoderControlPlane` in the same namespace with `status.operatorAccessReady: true`. The operator user must be a site owner. Coder answers `403` to anyone else on the provider routes. - An API key for the provider, for example an OpenAI key. +- Coder v2.37.2 or later. The Kind end-to-end test runs Coder v2.37.2, and the operator is built against the v2.37.3 SDK. Coder has the provider API since v2.34.0, but versions before v2.37.2 are not tested. +- An operator release after v0.1.3. The `dist/install.yaml` on `main` installs the `CoderAIProvider` CRD, but its `ghcr.io/coder/coder-k8s:latest` image is v0.1.3 until the next release, and v0.1.3 has no `CoderAIProvider` controller. Install from a release tag that includes `CoderAIProvider`, and pin the image to the same version (see [Deploy a Coder Control Plane](../tutorials/getting-started.md#1-install-the-operator)). ## 1. Prepare the key Secret diff --git a/hack/e2e-workspace-lifecycle.sh b/hack/e2e-workspace-lifecycle.sh index d8d35ea65..65fb15526 100755 --- a/hack/e2e-workspace-lifecycle.sh +++ b/hack/e2e-workspace-lifecycle.sh @@ -46,7 +46,7 @@ k() { kubectl --request-timeout=30s "$@"; } # every non-streaming kubectl reques cleanup() { local p for p in "${BG_PIDS[@]}"; do kill "$p" 2>/dev/null || true; done - rm -f "$WORK/coder.hdr" "$WORK/coder-body.json" "$WORK/tester.json" + rm -f "$WORK/coder.hdr" "$WORK/coder-body.json" "$WORK/tester.json" "$WORK"/ai-key-* "$WORK"/secret-*.json "$WORK/ai-leak.txt" [[ $RESULT == PASS || -z $CURRENT ]] || CASES+=("case: $CURRENT = FAILED") # Sanitized evidence receipt: identifiers only, never the token. printf '%s\n' "=== RECEIPT ($RESULT) ===" "source_sha=$SOURCE_SHA" "run_id=${GITHUB_RUN_ID:-unset}" \ @@ -561,11 +561,136 @@ step "tester API keys after the template tests (plan A4: recorded, not enforced) KEYS_AFTER=$(db_count api_keys user_id "$TESTER_ID") || fail "cannot count the api_keys rows of the tester after the tests" log "tester api_keys: before=$KEYS_BEFORE after=$KEYS_AFTER delta=$((KEYS_AFTER - KEYS_BEFORE))" +# CoderAIProvider phases (#154, plan 8.2 P1-P4, P6). Keys are synthetic and stay in 0600 files: never argv or output. +AI=e2e-openai AI_SECRET=e2e-ai-key AI_ORIGIN=http://ai-upstream.invalid:8080 +gone_obj() { ! k -n "$NS" get "$1" "$2" -o name >/dev/null 2>"$WORK/obj.err" && grep -q NotFound "$WORK/obj.err"; } +new_key() { # : 48 random hex characters + head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/$1" && [[ $(wc -c <"$WORK/$1") == 48 ]] || return 1 + [[ ${GITHUB_ACTIONS:-} != true ]] || echo "::add-mask::$(<"$WORK/$1")" +} +key_secret() { # : an Opaque Secret for $AI_ORIGIN that holds the key file + jq -n --arg n "$2" --arg ns "$NS" --rawfile v "$WORK/$3" --argjson l "$4" --arg o "$AI_ORIGIN" '{apiVersion: "v1", kind: "Secret", + metadata: {name: $n, namespace: $ns, labels: $l, annotations: {"coder.com/credential-destinations": $o}}, type: "Opaque", + data: {"api-key": ($v | @base64)}}' >"$WORK/secret-$2.json" && k "$1" -f "$WORK/secret-$2.json" >/dev/null +} +ai_provider() { # [baseURL]: kubectl apply of an openai-compat CoderAIProvider + jq -n --arg n "$1" --arg ns "$NS" --arg s "$2" --arg k "$3" --arg u "${4:-$AI_ORIGIN/v1}" '{apiVersion: "coder.com/v1alpha1", + kind: "CoderAIProvider", metadata: {name: $n, namespace: $ns}, spec: {controlPlaneRef: {name: "coder"}, type: "openai-compat", + baseURL: $u, displayName: ("E2E " + $n), apiKeySecretRef: {name: $s, key: $k}}}' >"$WORK/ai-$1.json" || fail "cannot render CoderAIProvider $1" + apply_manifest "$WORK/ai-$1.json" +} +AI_SEEN="" AI_S="" +ai_state() { # : reads the object into $WORK/ai-.state.json; sets AI_S to " stalled=" + k -n "$NS" get coderaiprovider "$1" -o json >"$WORK/ai-$1.state.json" 2>"$WORK/ai.err" || return 1 + AI_S=$(jq -r '.metadata.generation as $g | [.status.conditions[]? | select(.observedGeneration == $g)] as $c | + "\([$c[] | select(.type == "Ready") | "\(.status) \(.reason)"][0] // "- -") stalled=\([$c[] | select(.type == "Stalled") | .status][0] // "-")"' \ + "$WORK/ai-$1.state.json") || return 1 + [[ "$1 $AI_S" == "$AI_SEEN" ]] || { AI_SEEN="$1 $AI_S" && log "AI provider $1: $AI_S"; } +} +ai_is() { # : true once Ready has them for the current generation; another stall fails at once + ai_state "$1" || return 1 + [[ $AI_S == "$2 $3 "* ]] && return 0 + [[ $AI_S != *stalled=True ]] || fail "AI provider $1 stalled with $AI_S, want Ready=$2 $3: $(jq -r '.status.conditions[0].message' "$WORK/ai-$1.state.json")" + return 1 +} +ai_matches() { # : Coder's provider $AI_ID is enabled with the spec and only that key; sets AI_KEY_ID + coder_api GET "/api/v2/ai/providers/$AI_ID" >"$WORK/ai-coder.json" || return 1 + AI_KEY_ID=$(jq -er --arg n "$AI" --arg d "$2" --arg u "$AI_ORIGIN/v1" --rawfile v "$WORK/$1" 'select(.name == $n and + .type == "openai-compat" and .enabled and .display_name == $d and .base_url == $u and (.api_keys | length) == 1 and + .api_keys[0].masked == $v[0:4] + "..." + $v[-4:]) | .api_keys[0].id' "$WORK/ai-coder.json") +} +ai_coder_seen() { jq -c 'del(.api_keys[]?.masked)' "$WORK/ai-coder.json"; } # the answer without key masks, for messages +ai_list() { coder_api GET /api/v2/ai/providers | jq -ce 'map({id, name, enabled, display_name, base_url, updated_at, + keys: [.api_keys[]?.id]}) | sort_by(.name)'; } + +step "CoderAIProvider: a provider on an opted-in Secret is Ready; Coder holds the spec and the masked key (plan 8.2 P1)" +T0=$SECONDS +new_key ai-key-1 || fail "cannot generate the first AI provider key" +key_secret create "$AI_SECRET" ai-key-1 '{"coder.com/credential": "true"}' || fail "cannot create Secret $AI_SECRET" +ai_provider "$AI" "$AI_SECRET" api-key +wait_until "AI provider $AI Ready" ai_is "$AI" True Applied +AI_ID=$(jq -er '.status.providerID' "$WORK/ai-$AI.state.json") || fail "AI provider $AI is Ready without status.providerID" +ai_matches ai-key-1 "E2E $AI" || fail "Coder provider $AI_ID does not match the spec and the first key: $(ai_coder_seen)" +FIRST_KEY_ID=$AI_KEY_ID +{ cat "$WORK/ai-key-1" && echo && jq -r '.api_keys[0].masked' "$WORK/ai-coder.json"; } >"$WORK/ai-leak.txt" +! grep -qFf "$WORK/ai-leak.txt" "$WORK/ai-$AI.state.json" || fail "CoderAIProvider $AI holds its key or the key mask" +phase ai-provider + +step "CoderAIProvider rotation: a new Secret value sends one new key; a metadata-only change sends none (plan 8.2 P2)" +T0=$SECONDS +new_key ai-key-2 || fail "cannot generate the second AI provider key" +key_secret replace "$AI_SECRET" ai-key-2 '{"coder.com/credential": "true"}' || fail "cannot write the second key to Secret $AI_SECRET" +rotated() { ai_matches ai-key-2 "E2E $AI" && [[ $AI_KEY_ID != "$FIRST_KEY_ID" ]]; } +wait_until "Coder to hold the second key of $AI" rotated +ROTATED_KEY_ID=$AI_KEY_ID +key_recorded() { ai_state "$AI" && jq -e --arg id "$ROTATED_KEY_ID" '.status.credential.keyID == $id' "$WORK/ai-$AI.state.json" >/dev/null; } +wait_until "AI provider $AI to record key $ROTATED_KEY_ID" key_recorded +SECRET_RV=$(k -n "$NS" annotate secret "$AI_SECRET" e2e.coder.com/touched=1 -o json | jq -er '.metadata.resourceVersion') || + fail "cannot annotate Secret $AI_SECRET" +rv_recorded() { ai_state "$AI" && jq -e --arg rv "$SECRET_RV" '.status.credential.secretResourceVersion == $rv' "$WORK/ai-$AI.state.json" >/dev/null; } +wait_until "AI provider $AI to record Secret resourceVersion $SECRET_RV" rv_recorded +ai_matches ai-key-2 "E2E $AI" || fail "Coder provider $AI_ID lost the second key: $(ai_coder_seen)" +[[ $AI_KEY_ID == "$ROTATED_KEY_ID" ]] || fail "a metadata-only Secret change sent a new key ($ROTATED_KEY_ID -> $AI_KEY_ID)" +k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-ai.txt" || fail "cannot read the operator log" +LEAKS=$({ cat "$WORK/ai-key-1" && echo && cat "$WORK/ai-key-2"; } | grep -cFf - "$WORK/operator-ai.txt" || true) +[[ $LEAKS == 0 ]] || fail "the operator log holds an AI provider key ($LEAKS lines)" +phase ai-rotation + +step "CoderAIProvider refusals: an unlabeled Secret, the operator token, a foreign origin and a taken name write nothing to Coder (plan 8.2 P3)" +T0=$SECONDS +coder_api POST /api/v2/ai/providers "$(jq -nc --arg u "$AI_ORIGIN/v1" '{type: "openai-compat", name: "e2e-taken", base_url: $u, enabled: false}')" \ + >/dev/null || fail "cannot create provider e2e-taken through the Coder API" +new_key ai-key-3 || fail "cannot generate the third AI provider key" +key_secret create e2e-ai-key-unlabeled ai-key-3 '{}' || fail "cannot create Secret e2e-ai-key-unlabeled" +AI_BEFORE=$(ai_list) || fail "cannot list the Coder AI providers" +ai_provider e2e-p3-unlabeled e2e-ai-key-unlabeled api-key +ai_provider e2e-p3-refused "$SECRET" "$KEY" +ai_provider e2e-p3-origin "$AI_SECRET" api-key http://other.invalid:8080/v1 +ai_provider e2e-taken "$AI_SECRET" api-key +for c in e2e-p3-unlabeled:CredentialNotOptedIn e2e-p3-refused:CredentialRefused e2e-p3-origin:CredentialDestinationNotAllowed e2e-taken:NameConflict; do + wait_until "AI provider ${c%%:*} to stall with ${c#*:}" ai_is "${c%%:*}" False "${c#*:}" + jq -e '(.status.providerID // "") == ""' "$WORK/ai-${c%%:*}.state.json" >/dev/null || fail "refused AI provider ${c%%:*} has a providerID" +done +AI_AFTER=$(ai_list) || fail "cannot list the Coder AI providers after the refusals" +[[ $AI_AFTER == "$AI_BEFORE" ]] || fail "a refused CoderAIProvider wrote to Coder: before=$AI_BEFORE after=$AI_AFTER" +k -n "$NS" delete coderaiprovider e2e-p3-unlabeled e2e-p3-refused e2e-p3-origin e2e-taken --wait=false >/dev/null || + fail "cannot delete the refused CoderAIProviders" +for c in e2e-p3-unlabeled e2e-p3-refused e2e-p3-origin e2e-taken; do wait_until "CoderAIProvider $c to disappear" gone_obj coderaiprovider "$c"; done +AI_AFTER=$(ai_list) || fail "cannot list the Coder AI providers after deleting the refused objects" +[[ $AI_AFTER == "$AI_BEFORE" ]] || fail "deleting a refused CoderAIProvider wrote to Coder: before=$AI_BEFORE after=$AI_AFTER" +phase ai-refusals + +step "CoderAIProvider kill switch: a disable in Coder stalls the object; a spec edit enables the provider again (plan 8.2 P4)" +T0=$SECONDS +coder_api PATCH "/api/v2/ai/providers/$AI_ID" '{"enabled":false}' >/dev/null || fail "cannot disable provider $AI in Coder" +k -n "$NS" annotate coderaiprovider "$AI" e2e.coder.com/poke=1 >/dev/null || fail "cannot annotate CoderAIProvider $AI" +wait_until "AI provider $AI to stall with DisabledExternally" ai_is "$AI" False DisabledExternally +coder_api GET "/api/v2/ai/providers/$AI_ID" >"$WORK/ai-coder.json" || fail "cannot read provider $AI_ID" +jq -e '.enabled == false' "$WORK/ai-coder.json" >/dev/null || fail "the controller undid the disable of provider $AI: $(ai_coder_seen)" +k -n "$NS" patch coderaiprovider "$AI" --type=merge -p '{"spec":{"displayName":"E2E edited"}}' >/dev/null || + fail "cannot edit spec.displayName of CoderAIProvider $AI" +wait_until "AI provider $AI Ready after the spec edit" ai_is "$AI" True Applied +ai_matches ai-key-2 "E2E edited" && [[ $AI_KEY_ID == "$ROTATED_KEY_ID" ]] || + fail "Coder provider $AI_ID is not enabled with the edited spec and the second key: $(ai_coder_seen)" +phase ai-kill-switch + +step "CoderAIProvider delete: deleting the object deletes the provider in Coder (plan 8.2 P6)" +T0=$SECONDS +k -n "$NS" delete coderaiprovider "$AI" --wait=false >/dev/null || fail "cannot delete CoderAIProvider $AI" +wait_until "CoderAIProvider $AI to disappear" gone_obj coderaiprovider "$AI" +if coder_api GET "/api/v2/ai/providers/$AI_ID" >/dev/null 2>"$WORK/ai-get.err"; then + fail "provider $AI ($AI_ID) still exists in Coder after its object was deleted" +fi +grep -q 'error: 404' "$WORK/ai-get.err" || fail "GET provider $AI_ID after the delete: $(head -c 300 "$WORK/ai-get.err")" +phase ai-delete + # Last: it scales Coder to zero and deletes namespace $NS. Later CI steps must not need either. step "namespace deletion releases a running test while Coder is unreachable (plan A1, A13.2)" tt_create e2e-ns-delete "$(param startup_delay 120)" || fail "cannot create template test e2e-ns-delete" tt_waiting() { tt_state e2e-ns-delete && [[ $TT_S == "Running WaitingForAgents "* ]]; } wait_until "template test e2e-ns-delete at Running/WaitingForAgents" tt_waiting +ai_provider e2e-ns-keep "$AI_SECRET" api-key # plan 8.2 P7: a Ready provider goes with the namespace +wait_until "AI provider e2e-ns-keep Ready" ai_is e2e-ns-keep True Applied # spec.replicas is the operator's desired state for deploy/coder: the operator keeps it at zero instead of undoing it. k -n "$NS" patch codercontrolplane coder --type=merge -p '{"spec":{"replicas":0}}' >/dev/null || fail "cannot scale Coder to zero" coder_down() { k -n "$NS" get deploy coder -o json | jq -e '.spec.replicas == 0 and (.status.replicas // 0) == 0' >/dev/null; } @@ -574,10 +699,10 @@ tt_unavailable() { tt_state e2e-ns-delete && [[ $TT_S == "Running CoderUnavailab wait_until "template test e2e-ns-delete to report CoderUnavailable" tt_unavailable T0=$SECONDS k delete namespace "$NS" --wait=false >/dev/null || fail "cannot delete namespace $NS" -gone_obj() { ! k -n "$NS" get "$1" "$2" -o name >/dev/null 2>"$WORK/obj.err" && grep -q NotFound "$WORK/obj.err"; } NS_SEEN="" -ns_gone() { # the namespace disappears (#209), and with it the test and the control plane +ns_gone() { # the namespace disappears (#209), and with it the test, the provider and the control plane tt_state e2e-ns-delete || true # logs the ControlPlaneGone release while the object still exists + ai_state e2e-ns-keep || true if k get namespace "$NS" -o json >"$WORK/ns.json" 2>"$WORK/ns.err"; then # Diagnostics from the first poll on, so a run shows NamespaceDeletionContentFailure (the aggregated # LIST errors while the control plane still exists): the phase and every True condition, on change. @@ -588,10 +713,11 @@ ns_gone() { # the namespace disappears (#209), and with it the test and the cont NS_SEEN=$seen return 1 fi - grep -q NotFound "$WORK/ns.err" && gone_obj codertemplatetest e2e-ns-delete && gone_obj codercontrolplane coder + grep -q NotFound "$WORK/ns.err" && gone_obj codertemplatetest e2e-ns-delete && gone_obj coderaiprovider e2e-ns-keep && + gone_obj codercontrolplane coder } -TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, control plane, and namespace $NS deleted (ControlPlaneGone release)" ns_gone +TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, provider, control plane, and namespace $NS deleted (ControlPlaneGone release)" ns_gone NS_DELETE_SECONDS=$((SECONDS - T0)) -log "namespace $NS deleted in ${NS_DELETE_SECONDS}s, after its test and control plane" +log "namespace $NS deleted in ${NS_DELETE_SECONDS}s, after its test, provider and control plane" CASES+=("case: $CURRENT = passed") && CURRENT="" && RESULT=PASS log "PASS: workspace lifecycle" diff --git a/hack/e2e-workspace-lifecycle_test.sh b/hack/e2e-workspace-lifecycle_test.sh index 7ac0f5caa..a81922ccc 100755 --- a/hack/e2e-workspace-lifecycle_test.sh +++ b/hack/e2e-workspace-lifecycle_test.sh @@ -54,8 +54,54 @@ tt_json() { # [conditions]: a CoderTemplateTest object for the jq -n --arg n "${pos[2]}" --arg p "$1" --arg r "$2" --argjson c "${3:-[]}" \ '{metadata: {name: $n}, status: {phase: $p, reason: $r, message: "stub", workspaceName: ("ktt-" + $n), workspaceID: ("ttws-" + $n), conditions: $c}}' } +ai_set() { local f=$1 && shift && jq "$@" "$f" >"$f.tmp" && mv "$f.tmp" "$f"; } # : edits a JSON file +ai_json() { # : the object with Ready and Stalled conditions of its generation + jq --arg r "$2" '.metadata.generation as $g | .status.conditions = [{type: "Ready", status: (if $r == "Applied" then "True" else "False" end), + reason: $r, message: "stub", observedGeneration: $g}, {type: "Stalled", status: (if $r == "Applied" or $r == "CoderUnavailable" then "False" + else "True" end), reason: $r, observedGeneration: $g}]' "$1" +} +ai_get() { # : one reconcile of a fake CoderAIProvider controller against the fake Coder providers in $S/aip + local cr=$S/cr/$1.json p=$S/aip/$1.json sf v rv m r=Applied + [[ -f $cr ]] || err NotFound "coderaiproviders.coder.com \"$1\" not found" + sf=$S/secrets/$(jq -r .spec.apiKeySecretRef.name "$cr").json + if [[ -f $S/ns-deleted ]]; then ai_json "$cr" ControlPlaneGone; [[ $SCENARIO == ai-ns-stuck ]] || rm "$cr"; return; fi + if [[ ! -f $sf ]]; then r=CredentialRefused # only the driver's Secrets are in the store; the operator token is refused + elif [[ $(jq -r '.metadata.labels["coder.com/credential"] // ""' "$sf") != true ]]; then r=CredentialNotOptedIn + elif [[ $(jq -r .spec.baseURL "$cr") != "$(jq -r '.metadata.annotations["coder.com/credential-destinations"]' "$sf")"/* ]]; then + r=CredentialDestinationNotAllowed && [[ $SCENARIO != ai-wrong-reason ]] || r=CredentialMalformed + elif [[ -f $p && $(jq -r '.status.providerID // ""' "$cr") == "" ]]; then r=NameConflict + [[ $SCENARIO != ai-conflict-writes ]] || ai_set "$p" '.updated_at = "changed"' + elif [[ $SCENARIO == ai-never-ready ]]; then r=CoderUnavailable + elif [[ -f $p && ($(jq .enabled "$p") == false || -f $S/ai-killed) && $(jq .metadata.generation "$cr") == "$(jq .status.appliedGeneration "$cr")" ]]; then + r=DisabledExternally && [[ $SCENARIO != ai-kill-undone ]] || { ai_set "$p" '.enabled = true' && touch "$S/ai-killed"; } + else # create or sync: spec values, and a new key ID when the value differs from the sent one (ai-meta-resend: on any Secret change) + [[ -f $p ]] || jq -n --arg n "$1" '{id: ("aip-" + $n), name: $n, type: "openai-compat", api_keys: [], updated_at: "1"}' >"$p" + ai_set "$p" --slurpfile c "$cr" '. + {enabled: true, display_name: $c[0].spec.displayName, base_url: $c[0].spec.baseURL}' + v=$(jq -r '.data["api-key"]' "$sf" | base64 -d) rv=$(jq -r .metadata.resourceVersion "$sf") + if [[ $(cat "$S/aisent-$1" 2>/dev/null) != "$v" || ($SCENARIO == ai-meta-resend && $(jq -r .status.credential.secretResourceVersion "$cr") != "$rv") ]]; then + id=key-$RANDOM$RANDOM && [[ $SCENARIO != ai-key-id-unchanged || ! -f $S/aisent-$1 ]] || id=$(jq -r '.api_keys[0].id' "$p") + m=${v:0:4}...${v: -4} && [[ $SCENARIO != ai-mask-wrong ]] || m=${v:0:3}...${v: -4} + ai_set "$p" --arg id "$id" --arg m "$m" '.api_keys = [{id: $id, masked: $m}]' && printf '%s' "$v" >"$S/aisent-$1" + fi + ai_set "$cr" --slurpfile p "$p" --arg rv "$rv" --arg sc "$SCENARIO" '.status += {providerID: $p[0].id, appliedGeneration: .metadata.generation, + credential: ({keyID: $p[0].api_keys[0].id, secretResourceVersion: $rv} + (if $sc == "ai-cr-leak" then {mask: $p[0].api_keys[0].masked} else {} end))}' + fi + ai_json "$cr" "$r" +} +ai_watch() { local f; for f in "$S"/cr/*.json; do [[ $(jq -r .spec.apiKeySecretRef.name "$f") != "$1" ]] || ai_get "$(basename "$f" .json)" >/dev/null; done; } case "${pos[0]}:${pos[1]:-}" in get:pods) cat "$S/pods.json" ;; + get:coderaiprovider) ai_get "${pos[2]}" ;; + delete:coderaiprovider) # the controller deletes only the Coder provider that it created (ai-delete-kept: none) + for n in "${pos[@]:2}"; do + [[ $SCENARIO == ai-delete-kept || $(jq -r '.status.providerID // ""' "$S/cr/$n.json") != "aip-$n" ]] || rm "$S/aip/$n.json" + rm "$S/cr/$n.json" + done; echo deleted ;; + patch:coderaiprovider) ai_set "$S/cr/${pos[2]}.json" --argjson p "$patch" '. * $p | .metadata.generation += 1'; echo patched ;; + annotate:coderaiprovider) ai_get "${pos[2]}" >/dev/null; echo annotated ;; + annotate:secret) f=$S/secrets/${pos[2]}.json + ai_set "$f" --arg a "${pos[3]}" '.metadata.annotations[$a | split("=")[0]] = ($a | split("=")[1]) | .metadata.resourceVersion |= (tonumber + 1 | tostring)' + ai_watch "${pos[2]}"; cat "$f" ;; patch:codercontrolplane) # templateTests.ownerUserID, then spec.replicas=0 in the namespace phase if [[ $patch == *ownerUserID* ]]; then [[ $SCENARIO != owner-patch-fails ]] || err Invalid "bad ownerUserID"; touch "$S/owner" else touch "$S/replicas0"; fi; echo "codercontrolplane.coder.com/coder patched" ;; @@ -113,7 +159,8 @@ case "${pos[0]}:${pos[1]:-}" in echo '{"status":{"operatorTokenSecretRef":{"name":"op-token","key":"token"}}}' ;; get:secret) printf '{"data":{"token":"%s"}}\n' "$(printf secret-token-value | base64)" ;; port-forward:*) echo $$ >"$S/pf.pid"; exec sleep 300 ;; - logs:*) echo "I1002 server started"; [[ $SCENARIO != log-leak ]] || echo "2026-10-02T10:00:00Z [info] [provisioner|Planning infrastructure] Terraform 1.14.0 ok" ;; + logs:*) echo "I1002 server started"; [[ $SCENARIO != ai-log-leak || ! -f $S/aisent-e2e-openai ]] || { cat "$S/aisent-e2e-openai"; echo; } + [[ $SCENARIO != log-leak ]] || echo "2026-10-02T10:00:00Z [info] [provisioner|Planning infrastructure] Terraform 1.14.0 ok" ;; get:) if [[ $raw == */codertemplatetests?watch=1* ]]; then # the e2e-ttl watch: passed (ttl-failed: failed), then deleted echo "I1003 round_trippers.go:553] GET https://127.0.0.1:6443$raw 200 OK in 2 milliseconds" >&2 @@ -149,6 +196,10 @@ case "${pos[0]}:${pos[1]:-}" in jq -n --arg av "$av" --argjson n "$n" '{items: [range(1; $n + 1) | ("tv-\(.)") as $id | {status: {id: $id, active: ($id == $av)}}]}' ;; create:) if [[ ${file##*/} == tt-*.json ]]; then echo 0 >"$S/tt-$(jq -r .metadata.name "$file")"; echo created; exit 0; fi + if [[ ${file##*/} == secret-*.json ]]; then # an AI provider key Secret; every key value is kept for the leak checks + jq -r '.data["api-key"] | @base64d' "$file" >>"$S/ai-keys.txt" + jq '.metadata.resourceVersion = "1"' "$file" >"$S/secrets/$(jq -r .metadata.name "$file").json"; echo created; exit 0 + fi if [[ $raw == */codertemplates/*/promote* ]]; then # promote-* scenarios break the activation read -r id av n st <"$S/template"; v=$(jq -r .spec.versionID "$file") if [[ $av == "$v" ]]; then r=AlreadyActive; [[ $SCENARIO != promote-repeat-writes ]] || st=$((st + 1)) @@ -179,6 +230,9 @@ case "${pos[0]}:${pos[1]:-}" in read -r id av n st <"$S/template" [[ $SCENARIO != reapply-new-version ]] || echo "$id $av $((n + 1)) $st" >"$S/template" echo "codertemplate/coder.e2e-template configured" + elif [[ ${file##*/} == ai-*.json ]]; then # a CoderAIProvider: created at generation 1 + f=$S/cr/$(jq -r .metadata.name "$file").json && [[ -f $f ]] || jq '.metadata.generation = 1 | .status = {}' "$file" >"$f" + echo "coderaiprovider configured" elif [[ $(jq -r .kind "$file") == CoderTemplate ]]; then # changed files: a new active version read -r id _ n st <"$S/template"; echo "$id tv-$((n + 1)) $((n + 1)) $((st + 1))" >"$S/template" echo "codertemplate/coder.e2e-template configured" @@ -190,6 +244,11 @@ case "${pos[0]}:${pos[1]:-}" in echo "coderworkspace configured" fi ;; replace:) + if [[ ${file##*/} == secret-*.json ]]; then # a key rotation + f=$S/secrets/$(jq -r .metadata.name "$file").json && jq -r '.data["api-key"] | @base64d' "$file" >>"$S/ai-keys.txt" + jq --slurpfile o "$f" '.metadata.resourceVersion = ($o[0].metadata.resourceVersion | tonumber + 1 | tostring)' "$file" >"$f.tmp" && mv "$f.tmp" "$f" + ai_watch "$(jq -r .metadata.name "$f")"; echo replaced; exit 0 + fi f=$(wsfile "$raw"); [[ -f $f ]] || err NotFound missing; cp -n "$file" "$S/first_update.json" [[ $SCENARIO == stale-update-accepted || $(jq -r .metadata.resourceVersion "$file") == "$(jq -r .metadata.resourceVersion "$f")" ]] || err Conflict "rv mismatch" @@ -251,6 +310,14 @@ case "$method $path" in a=connected/ready && case $SCENARIO in agent-start-error) a=connected/start_error ;; agent-never-ready) a=connecting/created ;; esac jq -n --arg u "$uid" --arg lb "$lb" --arg a "$a" '{id: $u, latest_build: {id: $lb, resources: [{agents: [{name: "main", status: ($a | split("/")[0]), lifecycle_state: ($a | split("/")[1])}]}]}}' ;; + "GET /api/v2/ai/providers") cat "$S"/aip/*.json 2>/dev/null | jq -s . ;; + "POST /api/v2/ai/providers") + f=$S/aip/$(jq -r .name <<<"$data").json && [[ ! -f $f ]] || { echo "curl: (22) The requested URL returned error: 409" >&2; exit 22; } + jq '. + {id: ("aip-" + .name), display_name: .name, api_keys: [], updated_at: "1"}' <<<"$data" >"$f" && cat "$f" ;; + "GET /api/v2/ai/providers/aip-"* | "PATCH /api/v2/ai/providers/aip-"*) # a provider by ID; PATCH bumps updated_at + f=$S/aip/${path#/api/v2/ai/providers/aip-}.json && [[ -f $f ]] || { echo "curl: (22) The requested URL returned error: 404" >&2; exit 22; } + [[ $method == GET ]] || { jq --argjson d "$data" '. + $d | .updated_at = "2"' "$f" >"$f.tmp" && mv "$f.tmp" "$f"; } + cat "$f" ;; "GET /api/v2/organizations") echo '[{"id":"org-2","is_default":false},{"id":"org-1","is_default":true}]' ;; "POST /api/v2/users") printf '%s' "$data" >"$S/tester-body.json" @@ -301,7 +368,7 @@ run_scenario() { local name=$1 endpoint_ip=${ENDPOINT_IP:-10.244.0.5} start=$SECONDS shift T=$BASE/$name S=$BASE/$name/state - mkdir -p "$S/ws" + mkdir -p "$S/ws" "$S/secrets" "$S/cr" "$S/aip" : >"$S/calls.log" && : >"$S/events" && echo 0 >"$S/counter" jq -n '{items: [ {metadata: {name: "coder-k8s-old", deletionTimestamp: "2026-09-23T00:00:00Z"}, spec: {containers: [{image: "ghcr.io/coder/coder-k8s:e2e"}]}, @@ -319,8 +386,8 @@ run_scenario() { SECS=$((SECONDS - start)) } -mutations() { grep -E '^kubectl .*((create|replace|delete) --raw|apply -f|create -f|patch |delete (namespace|pod|codertemplatetest) )|^curl .*-X (PATCH|POST|PUT)' "$S/calls.log" | - awk '/^kubectl/ {for (i = 2; i <= NF; i++) if ($i ~ /^(create|replace|delete|apply|patch)$/) {print "kubectl " $i; next}} +mutations() { grep -E '^kubectl .*((create|replace|delete) --raw|apply -f|(create|replace) -f|patch |annotate |delete (namespace|pod|codertemplatetest|coderaiprovider) )|^curl .*-X (PATCH|POST|PUT|DELETE)' "$S/calls.log" | + awk '/^kubectl/ {for (i = 2; i <= NF; i++) if ($i ~ /^(create|replace|delete|apply|patch|annotate)$/) {print "kubectl " $i; next}} /^curl/ {for (i = 2; i <= NF; i++) if ($i == "-X") {print "curl " $(i + 1); next}}' | paste -sd, -; } VERSIONS="kubectl apply,kubectl apply,kubectl create,kubectl create,kubectl create" # template, second version, promote v1, repeat, dry-run v2 APPLIES="$VERSIONS,kubectl apply,kubectl apply,kubectl apply" # then workspace, identical template and workspace re-apply @@ -352,8 +419,13 @@ AGENT="$LIFECYCLE,curl POST,curl PUT,kubectl apply,kubectl create,kubectl delete TESTS="$AGENT,kubectl patch,kubectl create,kubectl create,kubectl create" # agent failure, bad parameter, restart (+ pod delete), mid-run (+ test delete), refused spec patch, TTL PHASED="$TESTS,kubectl create,kubectl create,kubectl create,kubectl delete,kubectl create,kubectl delete,kubectl patch,kubectl create" -check "mutation order: lifecycle, tester, agent, owner, three pass tests, namespace test, Coder to zero, namespace delete" \ - no_mutations_after "$PHASED,kubectl create,kubectl patch,kubectl delete" +# AI providers: P1 Secret and provider; P2 rotation and annotation; P3 e2e-taken in Coder, Secret, four refused objects; their delete; +# P4 disable in Coder, poke, spec edit; P6 delete. The namespace phase adds the provider e2e-ns-keep before Coder goes to zero. +P1M="kubectl create,kubectl apply" && P2M="$P1M,kubectl replace,kubectl annotate" +P3M="$P2M,curl POST,kubectl create,kubectl apply,kubectl apply,kubectl apply,kubectl apply" && P3D="$P3M,kubectl delete" +AIP="$P3D,curl PATCH,kubectl annotate,kubectl patch,kubectl delete" +check "mutation order: lifecycle, tester, agent, owner, tests, AI providers, namespace test and provider, Coder to zero, namespace delete" \ + no_mutations_after "$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete" check "stale requests carry the genuine pre-rename token; rename changed it" eval 'grep -qx "rv_pre_rename=2" "$T/work/receipt.txt" && grep -qx "rv_post_rename=2-renamed" "$T/work/receipt.txt" && jq -e ".metadata.resourceVersion == \"2\"" "$T/work/stale-update.json" >/dev/null' check "template applied with the long request timeout, then re-applied before any lifecycle mutation" eval '[[ $(grep -c -- "--request-timeout=600s apply -f config/e2e/codertemplate.yaml" "$S/calls.log") -eq 2 ]]' @@ -368,9 +440,9 @@ check "background port-forward and watch stopped" bg_stopped check "every non-streaming kubectl request carries --request-timeout=30s" \ eval '! grep "^kubectl" "$S/calls.log" | grep -v -e "watch=1" -e port-forward | grep -qvE -- "--request-timeout=[0-9]+s "' check "recreate only after the delete job succeeded" eval '[[ $(grep -n include_deleted "$S/calls.log" | head -1 | cut -d: -f1) -lt $(grep -n "create --raw .*ws-e2e-lifecycle-renamed.json" "$S/calls.log" | cut -d: -f1) ]]' -check "receipt: source, run, version, identity, UIDs, 28 passed cases" eval 'grep -q "=== RECEIPT (PASS) ===" "$T/out" && grep -qx "source_sha=0123abc" "$T/work/receipt.txt" && - grep -qx "run_id=42" "$T/work/receipt.txt" && grep -qx "coder_version=v2.37.2+eb69e27" "$T/work/receipt.txt" && grep -qx "uid1=uid-3" "$T/work/receipt.txt" && [[ $(grep -c "= passed$" "$T/work/receipt.txt") -eq 28 ]]' -check "remaining phases: durations recorded; dry run, restart, mid-run, and TTL calls as planned" eval 'grep -Eqx "phase_seconds=agent-failure=[0-9]+s bad-parameter=[0-9]+s restart=[0-9]+s delete-mid-run=[0-9]+s immutability-and-dry-run=[0-9]+s ttl=[0-9]+s" "$T/work/receipt.txt" && +check "receipt: source, run, version, identity, UIDs, 33 passed cases" eval 'grep -q "=== RECEIPT (PASS) ===" "$T/out" && grep -qx "source_sha=0123abc" "$T/work/receipt.txt" && + grep -qx "run_id=42" "$T/work/receipt.txt" && grep -qx "coder_version=v2.37.2+eb69e27" "$T/work/receipt.txt" && grep -qx "uid1=uid-3" "$T/work/receipt.txt" && [[ $(grep -c "= passed$" "$T/work/receipt.txt") -eq 33 ]]' +check "remaining phases: durations recorded; dry run, restart, mid-run, and TTL calls as planned" eval 'grep -Eqx "phase_seconds=agent-failure=[0-9]+s bad-parameter=[0-9]+s restart=[0-9]+s delete-mid-run=[0-9]+s immutability-and-dry-run=[0-9]+s ttl=[0-9]+s ai-provider=[0-9]+s ai-rotation=[0-9]+s ai-refusals=[0-9]+s ai-kill-switch=[0-9]+s ai-delete=[0-9]+s" "$T/work/receipt.txt" && grep -q "create --dry-run=server -f .*tt-e2e-dry-run.json" "$S/calls.log" && grep -q "FROM workspaces WHERE name = .ktt-d0d0d0d0d0d0d0d0d0d0d0d0d0d0." "$S/calls.log" && grep -q "coder-system delete pod coder-k8s-new" "$S/calls.log" && grep -q "workspaces/ttws-e2e-midrun?include_deleted=true" "$S/calls.log" && jq -e ".spec.ttlSecondsAfterFinished == 0" "$T/work/tt-e2e-ttl.json" >/dev/null && jq -e ".spec.parameters == [{name: \"fail\", value: \"notabool\"}]" "$T/work/tt-e2e-bad-param.json" >/dev/null' @@ -398,6 +470,12 @@ check "agent workspace: tester-owned, from the agent template, ready time logged jq -e ".metadata.name == \"coder.e2e-tester.e2e-agent\" and .spec.templateName == \"e2e-agent\"" "$T/work/ws-e2e-agent.json" >/dev/null && out_has "agent time-to-ready: " && out_has "agents: main=connected/ready" && grep -q "^agent_ready_seconds=[0-9]" "$T/work/receipt.txt" && grep -q "workspaces/uid-4?include_deleted=true" "$S/calls.log"' +check "AI provider keys: three, never printed or in argv, files removed" eval '[[ $(grep -c . "$S/ai-keys.txt") == 3 ]] && + ! grep -qFf "$S/ai-keys.txt" "$T/out" "$S/calls.log" "$T/work/receipt.txt" && ! compgen -G "$T/work/ai-key-*" && ! compgen -G "$T/work/secret-*.json"' +check "AI provider phases: each refusal reason seen; e2e-taken kept in Coder; e2e-openai deleted; e2e-ns-keep released" eval 'out_has "AI provider e2e-p3-unlabeled: False CredentialNotOptedIn" && + out_has "AI provider e2e-p3-refused: False CredentialRefused" && out_has "AI provider e2e-p3-origin: False CredentialDestinationNotAllowed" && + out_has "AI provider e2e-taken: False NameConflict" && out_has "AI provider e2e-openai: False DisabledExternally" && out_has "AI provider e2e-ns-keep: False ControlPlaneGone" && + [[ -f $S/aip/e2e-taken.json && ! -f $S/aip/e2e-openai.json ]] && jq -e ".spec.apiKeySecretRef == {name: \"op-token\", key: \"token\"}" "$T/work/ai-e2e-p3-refused.json" >/dev/null' echo "TEST image-mismatch: serving image differs from built image" run_scenario image-mismatch SERVING_ID="$OTHER"; summary @@ -587,9 +665,9 @@ tt-failed|template test e2e-pass-1 failed: Failed AgentStartError|$AGENT,kubectl ws-rows-2|expected exactly one workspaces row named ktt-e2e-pass-1 (deleted rows included), found 2|$TESTS coder-rolls|setting templateTests.ownerUserID rolled deploy/coder|$TESTS keys-after-fails|cannot count the api_keys rows of the tester after the tests|$PHASED -coder-stays-up|timed out after 3s waiting for: deploy/coder without pods|$PHASED,kubectl create,kubectl patch -ns-delete-stuck|timed out after 2s waiting for: test, control plane, and namespace coder deleted|$PHASED,kubectl create,kubectl patch,kubectl delete -ns-stays-terminating|timed out after 2s waiting for: test, control plane, and namespace coder deleted|$PHASED,kubectl create,kubectl patch,kubectl delete +coder-stays-up|timed out after 3s waiting for: deploy/coder without pods|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch +ns-delete-stuck|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete +ns-stays-terminating|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete fail-ignored|template test e2e-agent-fail ended Succeeded Succeeded deleted=True/Deleted, want Failed AgentStartError|$TESTS,kubectl create badparam-created|template test e2e-bad-param: expected 0 workspaces rows named ktt-e2e-bad-param (deleted rows included), found 1|$TESTS,kubectl create,kubectl create restart-failed|template test e2e-restart failed: Failed AgentStartError|$TESTS,kubectl create,kubectl create,kubectl create,kubectl delete @@ -599,6 +677,25 @@ dryrun-persists|the server dry run created template test e2e-dry-run|$TESTS,kube ttl-failed|template test e2e-ttl was removed before it passed|$PHASED CASES +# name|expected message|mutations: each CoderAIProvider phase failure (plan 8.2) stops before the next mutation. +while IFS='|' read -r name msg muts; do + echo "TEST $name (CoderAIProvider phases)" + run_scenario "$name" E2E_NAMESPACE_DELETE_TIMEOUT_SECONDS=2