From 1bc84d5093da09984f6eb93724ed3524e36683c3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 14 Oct 2026 13:07:31 +0000 Subject: [PATCH] test: add the CoderAIGatewayKey Kind E2E phases and the #348 follow-ups The Kind E2E runs Coder without a license, so the new phases test the unlicensed path of the CoderAIGatewayKey controller (#156, plan 9): - gate: no aibridge entitlement, and the key LIST answers 403 Premium - without the control plane opt-in: Stalled NotAllowed, no Secret - with the opt-in: Stalled LicenseRequired, still no Secret - an existing unlabeled Secret: Stalled SecretNotOwned, Secret unchanged - leak check of the operator log and both objects - delete: both finalizers go within 30 s - throughout: 0 ai_gateway_keys rows, nextSerial 0, no current or pending key Fixes #348: - the provider delete phase (P6) requires Coder's own 404 message - the e2e path filter adds credential_checks.go and credentialsecret - the namespace phase (P7) checks the ControlPlaneGone release log line of AI provider e2e-ns-keep The delete retry test also covers Coder's "Route not found." answer. Change-Id: I0329b429eb54192be174daf857618a679b8e08aa Signed-off-by: Thomas Kosiewski --- .github/workflows/ci.yaml | 2 + hack/e2e-workspace-lifecycle.sh | 131 ++++++++++++++++-- hack/e2e-workspace-lifecycle_test.sh | 101 +++++++++++--- .../coderaigatewaykey_delete_test.go | 8 ++ 4 files changed, 213 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 645ec140..41ba3dd9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -89,6 +89,8 @@ jobs: - 'internal/controller/coderaigatewaykey*' - 'internal/controller/owned_secret.go' - 'internal/controller/reserved_secret_names.go' + - 'internal/controller/credential_checks.go' + - 'internal/credentialsecret/**' # Shared Coder access that CoderTemplateTest and later pool controllers call. - 'internal/controller/controlplane_access.go' - 'internal/controller/coder_access.go' diff --git a/hack/e2e-workspace-lifecycle.sh b/hack/e2e-workspace-lifecycle.sh index 65fb1552..6480d6e2 100755 --- a/hack/e2e-workspace-lifecycle.sh +++ b/hack/e2e-workspace-lifecycle.sh @@ -46,7 +46,7 @@ k() { kubectl --request-timeout=30s "$@"; } # every non-streaming kubectl reques cleanup() { local p for p in "${BG_PIDS[@]}"; do kill "$p" 2>/dev/null || true; done - rm -f "$WORK/coder.hdr" "$WORK/coder-body.json" "$WORK/tester.json" "$WORK"/ai-key-* "$WORK"/secret-*.json "$WORK/ai-leak.txt" + rm -f "$WORK/coder.hdr" "$WORK/coder-body.json" "$WORK/tester.json" "$WORK"/ai-key-* "$WORK"/secret-*.json "$WORK/ai-leak.txt" "$WORK/coder-raw.json" [[ $RESULT == PASS || -z $CURRENT ]] || CASES+=("case: $CURRENT = FAILED") # Sanitized evidence receipt: identifiers only, never the token. printf '%s\n' "=== RECEIPT ($RESULT) ===" "source_sha=$SOURCE_SHA" "run_id=${GITHUB_RUN_ID:-unset}" \ @@ -102,6 +102,10 @@ coder_api() { # [json-body]: the token and the body reach curl t [[ $# -lt 3 ]] || { printf '%s' "$3" >"$WORK/coder-body.json" && args+=(--data @"$WORK/coder-body.json"); } curl "${args[@]}" "$CODER_URL$2" } +coder_status() { # : a GET without -f that prints the HTTP status; the body goes to $WORK/coder-raw.json + curl -sS --max-time 30 -o "$WORK/coder-raw.json" -w '%{http_code}' -H @"$WORK/coder.hdr" "$CODER_URL$1" +} +CODER_NOT_FOUND="Resource not found or you do not have access to this resource" # Coder's own 404 message (#325) # build_done : true once the build reached status; fails on terminal failure. build_done() { @@ -423,13 +427,16 @@ ws_delete "$AGENT_NAME" "$AGENT_UID" >/dev/null || fail "cannot delete the agent wait_until "delete build of $AGENT_NAME (404)" gone "$AGENT_NAME" wait_until "delete job of $AGENT_UID to succeed" delete_succeeded "$AGENT_UID" -# db_count : rows of a Coder table (deleted rows included) through psql in the CNPG +# db_count
[ ]: rows of a Coder table (deleted rows included) through psql in the CNPG # primary. A read-only count: it never selects key values or hashes. db_count() { - local pod - [[ $3 =~ ^[A-Za-z0-9-]+$ ]] || fail "assertion failed: unsafe value for $1.$2: '$3'" + local pod where="" + if (($# > 1)); then + [[ $3 =~ ^[A-Za-z0-9-]+$ ]] || fail "assertion failed: unsafe value for $1.$2: '$3'" + where=" WHERE $2 = '$3'" + fi pod=$(k -n "$NS" get cluster coder-db -o json | jq -er '.status.currentPrimary') || return 1 - k -n "$NS" exec "$pod" -c postgres -- psql -d coder -XtAc "SELECT count(*) FROM $1 WHERE $2 = '$3'" | grep -Ex '[0-9]+' + k -n "$NS" exec "$pod" -c postgres -- psql -d coder -XtAc "SELECT count(*) FROM $1$where" | grep -Ex '[0-9]+' } tt_create() { # [parameters JSON] [spec JSON] [create flags]: a test of the agent template's active version jq -n --arg n "$1" --arg ns "$NS" --arg t "$ORG.$AGENT_TEMPLATE" --argjson p "${2:-[]}" --argjson x "${3:-"{}"}" '{apiVersion: "coder.com/v1alpha1", @@ -678,12 +685,115 @@ step "CoderAIProvider delete: deleting the object deletes the provider in Coder T0=$SECONDS k -n "$NS" delete coderaiprovider "$AI" --wait=false >/dev/null || fail "cannot delete CoderAIProvider $AI" wait_until "CoderAIProvider $AI to disappear" gone_obj coderaiprovider "$AI" -if coder_api GET "/api/v2/ai/providers/$AI_ID" >/dev/null 2>"$WORK/ai-get.err"; then - fail "provider $AI ($AI_ID) still exists in Coder after its object was deleted" -fi -grep -q 'error: 404' "$WORK/ai-get.err" || fail "GET provider $AI_ID after the delete: $(head -c 300 "$WORK/ai-get.err")" +CODE=$(coder_status "/api/v2/ai/providers/$AI_ID") || fail "cannot call GET provider $AI_ID after the delete" +[[ $CODE != 200 ]] || fail "provider $AI ($AI_ID) still exists in Coder after its object was deleted" +# Only Coder's own 404 proves the delete (#348): a proxy or a wrong route also answers 404. +jq -e --arg c "$CODE" --arg m "$CODER_NOT_FOUND" '$c == "404" and .message == $m' "$WORK/coder-raw.json" >/dev/null 2>&1 || + fail "GET provider $AI_ID after the delete answered HTTP $CODE without Coder's own 404 message: $(head -c 300 "$WORK/coder-raw.json")" phase ai-delete +# CoderAIGatewayKey phases (#156, plan 9). Kind runs Coder without a license, so they test the unlicensed path only: +# the controller stalls and sends no key create. The synthetic Secret value stays in a 0600 file: never argv or output. +GW_SEEN="" GW_S="" +gw_key() { # : kubectl apply of a CoderAIGatewayKey of control plane coder + jq -n --arg n "$1" --arg ns "$NS" --arg s "$2" '{apiVersion: "coder.com/v1alpha1", kind: "CoderAIGatewayKey", + metadata: {name: $n, namespace: $ns}, spec: {controlPlaneRef: {name: "coder"}, secretName: $s}}' >"$WORK/gw-$1.json" || + fail "cannot render CoderAIGatewayKey $1" + apply_manifest "$WORK/gw-$1.json" +} +gw_state() { # : reads the object into $WORK/gw-.state.json; sets GW_S to " " of the current generation + k -n "$NS" get coderaigatewaykey "$1" -o json >"$WORK/gw-$1.state.json" 2>"$WORK/gw.err" || return 1 + GW_S=$(jq -r '.metadata.generation as $g | if .status.observedGeneration == $g then [.status.conditions[]? | + select(.type == "Stalled" and .observedGeneration == $g) | "\(.status) \(.reason)"][0] // "- -" else "- -" end' "$WORK/gw-$1.state.json") || return 1 + [[ "$1 $GW_S" == "$GW_SEEN" ]] || { GW_SEEN="$1 $GW_S" && log "AI Gateway key $1: $GW_S"; } +} +gw_is() { # [earlier reason]: true once Stalled=True has reason for the current generation; another stall fails at once + gw_state "$1" || return 1 + [[ $GW_S == "True $2" ]] && return 0 + [[ $GW_S != True* || $GW_S == "True ${3:-}" ]] || + fail "AI Gateway key $1 stalled with $GW_S, want $2: $(jq -r '[.status.conditions[] | select(.type == "Stalled")][0].message' "$WORK/gw-$1.state.json")" + return 1 +} +gw_rows() { # Coder holds no AI Gateway key: the ai_gateway_keys table has 0 rows + local n + n=$(db_count ai_gateway_keys) || fail "cannot count the ai_gateway_keys rows" + [[ $n == 0 ]] || fail "Coder holds $n ai_gateway_keys rows, want 0" +} +gw_clean() { # : the object sent no create (nextSerial 0, no current or pending key), and Coder holds no key + jq -e '(.status.nextSerial // 0) == 0 and .status.currentKey == null and .status.pendingKey == null' "$WORK/gw-$1.state.json" >/dev/null || + fail "AI Gateway key $1 has key state: nextSerial=$(jq -r '.status.nextSerial // 0' "$WORK/gw-$1.state.json")" + gw_rows +} +secret_seen() { # : resourceVersion, ownerReferences and a SHA-256 of the data, never the data itself + local s + s=$(k -n "$NS" get secret "$1" -o json) || return 1 + printf '%s data=%s\n' "$(jq -r '"rv=\(.metadata.resourceVersion) owners=\(.metadata.ownerReferences // [] | tojson)"' <<<"$s")" \ + "$(jq -j '.data | tojson' <<<"$s" | sha256sum | cut -d' ' -f1)" +} + +step "CoderAIGatewayKey gate: unlicensed Coder has no aibridge feature and answers the key LIST 403 Premium (plan 9.1)" +T0=$SECONDS +coder_api GET /api/v2/entitlements >"$WORK/entitlements.json" || fail "cannot read the Coder entitlements" +jq -e '(.features | type) == "object" and (.features.aibridge.enabled // false) == false' "$WORK/entitlements.json" >/dev/null || + fail "Coder has the aibridge feature enabled: the AI Gateway key phases test the unlicensed path only" +CODE=$(coder_status /api/v2/ai-gateway/keys) || fail "cannot call GET /api/v2/ai-gateway/keys" +jq -e --arg c "$CODE" '$c == "403" and (.message | endswith("is a Premium feature. Contact sales!"))' "$WORK/coder-raw.json" >/dev/null 2>&1 || + fail "GET /api/v2/ai-gateway/keys answered HTTP $CODE, want 403 with the Premium message: $(jq -r 'if type == "object" then .message // "no message" else "a body without a message" end' "$WORK/coder-raw.json" 2>/dev/null | head -c 300)" +gw_rows +phase gw-gate + +step "CoderAIGatewayKey without the control plane opt-in stalls NotAllowed and creates no Secret (plan 9.2)" +T0=$SECONDS +gw_key e2e-gw-off e2e-gw-key +wait_until "AI Gateway key e2e-gw-off to stall with NotAllowed" gw_is e2e-gw-off NotAllowed +gone_obj secret e2e-gw-key || fail "Secret e2e-gw-key exists while AI Gateway key e2e-gw-off is NotAllowed" +gw_clean e2e-gw-off +phase gw-not-allowed + +# The opt-in stays on: no CoderAIGatewayKey is left when the namespace phase runs. +step "CoderAIGatewayKey with the opt-in stalls LicenseRequired: the LIST fails before any Secret claim (plan 9.3)" +T0=$SECONDS +k -n "$NS" patch codercontrolplane coder --type=merge -p '{"spec":{"aiGateway":{"allowKeys":true}}}' >/dev/null || + fail "cannot set aiGateway.allowKeys on codercontrolplane coder" +wait_until "AI Gateway key e2e-gw-off to stall with LicenseRequired" gw_is e2e-gw-off LicenseRequired NotAllowed +gone_obj secret e2e-gw-key || fail "Secret e2e-gw-key exists while AI Gateway key e2e-gw-off is LicenseRequired" +gw_clean e2e-gw-off +phase gw-license-required + +step "CoderAIGatewayKey refuses an existing Secret that it does not own: SecretNotOwned, the Secret is unchanged (plan 9.4)" +T0=$SECONDS +v=$(head -c 96 /dev/urandom | base64 -w0 | tr -dc 'A-Za-z0-9') && printf '%s' "${v:0:43}" >"$WORK/ai-key-gw" && unset v && + [[ $(wc -c <"$WORK/ai-key-gw") == 43 ]] || fail "cannot generate the synthetic AI Gateway key value" +[[ ${GITHUB_ACTIONS:-} != true ]] || echo "::add-mask::$(<"$WORK/ai-key-gw")" +key_secret create e2e-gw-hijack ai-key-gw '{}' || fail "cannot create Secret e2e-gw-hijack" +HIJACK_SEEN=$(secret_seen e2e-gw-hijack) || fail "cannot read Secret e2e-gw-hijack" +gw_key e2e-gw-hijack e2e-gw-hijack +wait_until "AI Gateway key e2e-gw-hijack to stall with SecretNotOwned" gw_is e2e-gw-hijack SecretNotOwned +[[ $(secret_seen e2e-gw-hijack) == "$HIJACK_SEEN" ]] || fail "Secret e2e-gw-hijack changed: before=[$HIJACK_SEEN] after=[$(secret_seen e2e-gw-hijack)]" +gw_clean e2e-gw-hijack +phase gw-secret-not-owned + +step "CoderAIGatewayKey leak check: no key-shaped value or Secret value in the operator log or the objects (plan 9.5)" +T0=$SECONDS +k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-gw.txt" || fail "cannot read the operator log" +k -n "$NS" get coderaigatewaykey e2e-gw-off e2e-gw-hijack -o yaml >"$WORK/gw-objects.yaml" || fail "cannot read the AI Gateway key objects" +# A key is a run of exactly 43 letters and digits. Only the log lines of the key controller are searched for one. +LEAKS=$({ grep -i coderaigatewaykey "$WORK/operator-gw.txt" || true; cat "$WORK/gw-objects.yaml"; } | + grep -cE '(^|[^A-Za-z0-9])[A-Za-z0-9]{43}([^A-Za-z0-9]|$)' || true) +LEAKS=$((LEAKS + $(cat "$WORK/operator-gw.txt" "$WORK/gw-objects.yaml" | grep -cFf "$WORK/ai-key-gw" || true))) +[[ $LEAKS == 0 ]] || fail "the operator log or an AI Gateway key object holds a key-shaped value or the Secret value ($LEAKS lines)" +phase gw-leak-check + +step "CoderAIGatewayKey delete: both finalizers go within 30 s without a key call to Coder (plan 9.6)" +T0=$SECONDS +k -n "$NS" delete coderaigatewaykey e2e-gw-off e2e-gw-hijack --wait=false >/dev/null || fail "cannot delete the AI Gateway key objects" +gw_gone() { gone_obj coderaigatewaykey e2e-gw-off && gone_obj coderaigatewaykey e2e-gw-hijack; } +TIMEOUT=$((TIMEOUT < 30 ? TIMEOUT : 30)) wait_until "AI Gateway keys e2e-gw-off and e2e-gw-hijack to disappear" gw_gone +gw_rows +[[ $(secret_seen e2e-gw-hijack) == "$HIJACK_SEEN" ]] || fail "deleting AI Gateway key e2e-gw-hijack changed Secret e2e-gw-hijack" +gone_obj secret e2e-gw-key || fail "Secret e2e-gw-key exists after the delete" +phase gw-delete + # Last: it scales Coder to zero and deletes namespace $NS. Later CI steps must not need either. step "namespace deletion releases a running test while Coder is unreachable (plan A1, A13.2)" tt_create e2e-ns-delete "$(param startup_delay 120)" || fail "cannot create template test e2e-ns-delete" @@ -718,6 +828,9 @@ ns_gone() { # the namespace disappears (#209), and with it the test, the provide } TIMEOUT=$NS_DELETE_TIMEOUT wait_until "test, provider, control plane, and namespace $NS deleted (ControlPlaneGone release)" ns_gone NS_DELETE_SECONDS=$((SECONDS - T0)) +k -n "$OP_NS" logs deploy/coder-k8s >"$WORK/operator-ns.txt" || fail "cannot read the operator log after the namespace deletion" +grep -F 'Releasing the finalizer without a delete in Coder' "$WORK/operator-ns.txt" | grep -F '"ControlPlaneGone"' | grep -qF '"e2e-ns-keep"' || + fail "the operator log has no ControlPlaneGone release of AI provider e2e-ns-keep (plan 8.2 P7)" log "namespace $NS deleted in ${NS_DELETE_SECONDS}s, after its test, provider and control plane" CASES+=("case: $CURRENT = passed") && CURRENT="" && RESULT=PASS log "PASS: workspace lifecycle" diff --git a/hack/e2e-workspace-lifecycle_test.sh b/hack/e2e-workspace-lifecycle_test.sh index a81922cc..fea1e4da 100755 --- a/hack/e2e-workspace-lifecycle_test.sh +++ b/hack/e2e-workspace-lifecycle_test.sh @@ -88,10 +88,23 @@ ai_get() { # : one reconcile of a fake CoderAIProvider controller against fi ai_json "$cr" "$r" } +gw_get() { # : one reconcile of a fake CoderAIGatewayKey controller against Coder without a license + local cr=$S/gw/$1.json sf r + [[ -f $cr ]] || err NotFound "coderaigatewaykeys.coder.com \"$1\" not found" + sf=$S/secrets/$(jq -r .spec.secretName "$cr").json + if [[ -f $sf ]]; then r=SecretNotOwned # only the driver's unlabeled Secret is in the store + [[ $SCENARIO != gw-hijack-written ]] || ai_set "$sf" '.metadata.ownerReferences = [{kind: "CoderAIGatewayKey", name: "e2e-gw-hijack"}]' + elif [[ ! -f $S/gw-allowed ]]; then r=NotAllowed && [[ $SCENARIO != gw-secret-created ]] || echo '{}' >"$sf" + else r=LicenseRequired && [[ $SCENARIO != gw-create-sent ]] || ai_set "$cr" '.status.nextSerial = 1'; fi + jq --arg r "$r" '.metadata.generation as $g | .status += {observedGeneration: $g, conditions: [{type: "Ready", status: "False", reason: $r, + observedGeneration: $g}, {type: "Stalled", status: "True", reason: $r, message: "stub", observedGeneration: $g}]}' "$cr" +} ai_watch() { local f; for f in "$S"/cr/*.json; do [[ $(jq -r .spec.apiKeySecretRef.name "$f") != "$1" ]] || ai_get "$(basename "$f" .json)" >/dev/null; done; } case "${pos[0]}:${pos[1]:-}" in get:pods) cat "$S/pods.json" ;; get:coderaiprovider) ai_get "${pos[2]}" ;; + get:coderaigatewaykey) if ((${#pos[@]} == 3)); then gw_get "${pos[2]}"; else for n in "${pos[@]:2}"; do cat "$S/gw/$n.json"; done; fi ;; + delete:coderaigatewaykey) [[ $SCENARIO == gw-finalizer-stuck ]] || for n in "${pos[@]:2}"; do rm "$S/gw/$n.json"; done; echo deleted ;; delete:coderaiprovider) # the controller deletes only the Coder provider that it created (ai-delete-kept: none) for n in "${pos[@]:2}"; do [[ $SCENARIO == ai-delete-kept || $(jq -r '.status.providerID // ""' "$S/cr/$n.json") != "aip-$n" ]] || rm "$S/aip/$n.json" @@ -102,8 +115,9 @@ case "${pos[0]}:${pos[1]:-}" in annotate:secret) f=$S/secrets/${pos[2]}.json ai_set "$f" --arg a "${pos[3]}" '.metadata.annotations[$a | split("=")[0]] = ($a | split("=")[1]) | .metadata.resourceVersion |= (tonumber + 1 | tostring)' ai_watch "${pos[2]}"; cat "$f" ;; - patch:codercontrolplane) # templateTests.ownerUserID, then spec.replicas=0 in the namespace phase + patch:codercontrolplane) # templateTests.ownerUserID, aiGateway.allowKeys, then spec.replicas=0 in the namespace phase if [[ $patch == *ownerUserID* ]]; then [[ $SCENARIO != owner-patch-fails ]] || err Invalid "bad ownerUserID"; touch "$S/owner" + elif [[ $patch == *allowKeys* ]]; then touch "$S/gw-allowed" else touch "$S/replicas0"; fi; echo "codercontrolplane.coder.com/coder patched" ;; get:deploy) # coder-rolls: the owner patch changes the pod template; coder-stays-up: replicas=0 never removes the pod r=1 g=1 && [[ ! -f $S/replicas0 ]] || r=0 && [[ $SCENARIO != coder-rolls || ! -f $S/owner ]] || g=2 @@ -120,6 +134,7 @@ case "${pos[0]}:${pos[1]:-}" in *"FROM workspaces"*"ktt-e2e-bad-param"*) [[ $SCENARIO == badparam-created ]] && echo 1 || echo 0 ;; *"FROM workspaces"*"ktt-d0d0"*) echo 0 ;; *"FROM workspaces"*) [[ $SCENARIO == ws-rows-2 ]] && echo 2 || echo 1 ;; + *"FROM ai_gateway_keys") [[ $SCENARIO == gw-db-row ]] && echo 1 || echo 0 ;; *) echo "stub kubectl: unexpected SQL: $all" >&2; exit 97 ;; esac ;; get:codertemplatetest) # e2e-pass-*: Pending, Running, then final; e2e-ns-delete: Running until Coder and the namespace go @@ -157,10 +172,16 @@ case "${pos[0]}:${pos[1]:-}" in get:endpointslices) cat "$S/endpoints.json" ;; get:codercontrolplane) [[ ! -f $S/ns-deleted || $SCENARIO == ns-delete-stuck ]] || err NotFound 'codercontrolplanes "coder" not found' echo '{"status":{"operatorTokenSecretRef":{"name":"op-token","key":"token"}}}' ;; - get:secret) printf '{"data":{"token":"%s"}}\n' "$(printf secret-token-value | base64)" ;; + get:secret) f=$S/secrets/${pos[2]}.json # the operator token, a driver Secret, or NotFound + if [[ -f $f ]]; then cat "$f"; elif [[ ${pos[2]} == op-token ]]; then printf '{"data":{"token":"%s"}}\n' "$(printf secret-token-value | base64)" + else err NotFound "secrets \"${pos[2]}\" not found"; fi ;; port-forward:*) echo $$ >"$S/pf.pid"; exec sleep 300 ;; logs:*) echo "I1002 server started"; [[ $SCENARIO != ai-log-leak || ! -f $S/aisent-e2e-openai ]] || { cat "$S/aisent-e2e-openai"; echo; } - [[ $SCENARIO != log-leak ]] || echo "2026-10-02T10:00:00Z [info] [provisioner|Planning infrastructure] Terraform 1.14.0 ok" ;; + [[ $SCENARIO != log-leak ]] || echo "2026-10-02T10:00:00Z [info] [provisioner|Planning infrastructure] Terraform 1.14.0 ok" + echo 'INFO Starting Controller {"controller": "coderaigatewaykey", "controllerKind": "CoderAIGatewayKey"}' + [[ $SCENARIO != gw-log-leak || ! -f $S/secrets/e2e-gw-hijack.json ]] || + echo "INFO stub {\"controller\": \"coderaigatewaykey\", \"value\": \"$(jq -r '.data["api-key"] | @base64d' "$S/secrets/e2e-gw-hijack.json")\"}" + [[ ! -f $S/ns-deleted || $SCENARIO == ai-ns-no-release ]] || echo 'INFO Releasing the finalizer without a delete in Coder: the provider can stay in the Coder database {"controller": "coderaiprovider", "reason": "ControlPlaneGone", "provider": "e2e-ns-keep"}' ;; get:) if [[ $raw == */codertemplatetests?watch=1* ]]; then # the e2e-ttl watch: passed (ttl-failed: failed), then deleted echo "I1003 round_trippers.go:553] GET https://127.0.0.1:6443$raw 200 OK in 2 milliseconds" >&2 @@ -233,6 +254,9 @@ case "${pos[0]}:${pos[1]:-}" in elif [[ ${file##*/} == ai-*.json ]]; then # a CoderAIProvider: created at generation 1 f=$S/cr/$(jq -r .metadata.name "$file").json && [[ -f $f ]] || jq '.metadata.generation = 1 | .status = {}' "$file" >"$f" echo "coderaiprovider configured" + elif [[ ${file##*/} == gw-*.json ]]; then # a CoderAIGatewayKey: created at generation 1 + f=$S/gw/$(jq -r .metadata.name "$file").json && [[ -f $f ]] || jq '.metadata.generation = 1 | .status = {}' "$file" >"$f" + echo "coderaigatewaykey configured" elif [[ $(jq -r .kind "$file") == CoderTemplate ]]; then # changed files: a new active version read -r id _ n st <"$S/template"; echo "$id tv-$((n + 1)) $((n + 1)) $((st + 1))" >"$S/template" echo "codertemplate/coder.e2e-template configured" @@ -282,10 +306,12 @@ cat >"$STUBS/curl" <<'STUB' set -euo pipefail S=$STUB_STATE echo "curl $*" >>"$S/calls.log" -method=GET data="" url="" +method=GET data="" url="" out="" while (($#)); do case $1 in -X) method=$2; shift 2 ;; + -o) out=$2; shift 2 ;; + -w) shift 2 ;; -H) [[ $2 != @* ]] || stat -c %a "${2#@}" >>"$S/file-modes"; shift 2 ;; --max-time) shift 2 ;; --data) data=$2; [[ $data != @* ]] || { stat -c %a "${data#@}" >>"$S/file-modes"; data=$(<"${data#@}"); }; shift 2 ;; @@ -294,7 +320,11 @@ while (($#)); do esac done path=/${url#http://*/} +answer() { printf '%s' "$2" >"$out"; printf '%s' "$1"; exit 0; } # : the answer to a call with -o and -w, without -f case "$method $path" in + "GET /api/v2/entitlements") en=false && [[ $SCENARIO != gw-entitled ]] || en=true + jq -n --argjson e "$en" '{features: {aibridge: {entitlement: "not_entitled", enabled: $e}}, has_license: false}' ;; + "GET /api/v2/ai-gateway/keys") [[ -n $out ]] || exit 97; answer 403 '{"message":"AI Gateway is a Premium feature. Contact sales!"}' ;; "GET /api/v2/buildinfo") printf '{"version":"%s"}\n' "${CODER_VERSION_REPORTED:-v2.37.2+eb69e27}" ;; "GET /api/v2/organizations/coder/templates/e2e-template") [[ -f $S/template ]] || exit 22; read -r id av _ st <"$S/template" @@ -315,7 +345,12 @@ case "$method $path" in f=$S/aip/$(jq -r .name <<<"$data").json && [[ ! -f $f ]] || { echo "curl: (22) The requested URL returned error: 409" >&2; exit 22; } jq '. + {id: ("aip-" + .name), display_name: .name, api_keys: [], updated_at: "1"}' <<<"$data" >"$f" && cat "$f" ;; "GET /api/v2/ai/providers/aip-"* | "PATCH /api/v2/ai/providers/aip-"*) # a provider by ID; PATCH bumps updated_at - f=$S/aip/${path#/api/v2/ai/providers/aip-}.json && [[ -f $f ]] || { echo "curl: (22) The requested URL returned error: 404" >&2; exit 22; } + f=$S/aip/${path#/api/v2/ai/providers/aip-}.json + if [[ -n $out ]]; then # ai-delete-foreign-404: a 404 that is not Coder's own + [[ ! -f $f ]] || answer 200 "$(<"$f")"; m="Resource not found or you do not have access to this resource" + [[ $SCENARIO != ai-delete-foreign-404 ]] || m="Route not found."; answer 404 "$(jq -nc --arg m "$m" '{message: $m}')" + fi + [[ -f $f ]] || { echo "curl: (22) The requested URL returned error: 404" >&2; exit 22; } [[ $method == GET ]] || { jq --argjson d "$data" '. + $d | .updated_at = "2"' "$f" >"$f.tmp" && mv "$f.tmp" "$f"; } cat "$f" ;; "GET /api/v2/organizations") echo '[{"id":"org-2","is_default":false},{"id":"org-1","is_default":true}]' ;; @@ -368,7 +403,7 @@ run_scenario() { local name=$1 endpoint_ip=${ENDPOINT_IP:-10.244.0.5} start=$SECONDS shift T=$BASE/$name S=$BASE/$name/state - mkdir -p "$S/ws" "$S/secrets" "$S/cr" "$S/aip" + mkdir -p "$S/ws" "$S/secrets" "$S/cr" "$S/aip" "$S/gw" : >"$S/calls.log" && : >"$S/events" && echo 0 >"$S/counter" jq -n '{items: [ {metadata: {name: "coder-k8s-old", deletionTimestamp: "2026-09-23T00:00:00Z"}, spec: {containers: [{image: "ghcr.io/coder/coder-k8s:e2e"}]}, @@ -386,7 +421,7 @@ run_scenario() { SECS=$((SECONDS - start)) } -mutations() { grep -E '^kubectl .*((create|replace|delete) --raw|apply -f|(create|replace) -f|patch |annotate |delete (namespace|pod|codertemplatetest|coderaiprovider) )|^curl .*-X (PATCH|POST|PUT|DELETE)' "$S/calls.log" | +mutations() { grep -E '^kubectl .*((create|replace|delete) --raw|apply -f|(create|replace) -f|patch |annotate |delete (namespace|pod|codertemplatetest|coderaiprovider|coderaigatewaykey) )|^curl .*-X (PATCH|POST|PUT|DELETE)' "$S/calls.log" | awk '/^kubectl/ {for (i = 2; i <= NF; i++) if ($i ~ /^(create|replace|delete|apply|patch|annotate)$/) {print "kubectl " $i; next}} /^curl/ {for (i = 2; i <= NF; i++) if ($i == "-X") {print "curl " $(i + 1); next}}' | paste -sd, -; } VERSIONS="kubectl apply,kubectl apply,kubectl create,kubectl create,kubectl create" # template, second version, promote v1, repeat, dry-run v2 @@ -424,8 +459,10 @@ PHASED="$TESTS,kubectl create,kubectl create,kubectl create,kubectl delete,kubec P1M="kubectl create,kubectl apply" && P2M="$P1M,kubectl replace,kubectl annotate" P3M="$P2M,curl POST,kubectl create,kubectl apply,kubectl apply,kubectl apply,kubectl apply" && P3D="$P3M,kubectl delete" AIP="$P3D,curl PATCH,kubectl annotate,kubectl patch,kubectl delete" -check "mutation order: lifecycle, tester, agent, owner, tests, AI providers, namespace test and provider, Coder to zero, namespace delete" \ - no_mutations_after "$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete" +# AI Gateway keys: the opted-out object, the opt-in patch, the unlabeled Secret and the second object, then the delete of both. +GW1="$PHASED,$AIP,kubectl apply" && GW2="$GW1,kubectl patch" && GW3="$GW2,kubectl create,kubectl apply" && GW="$GW3,kubectl delete" +check "mutation order: lifecycle, tester, agent, owner, tests, AI providers, AI Gateway keys, namespace test and provider, Coder to zero, namespace delete" \ + no_mutations_after "$GW,kubectl create,kubectl apply,kubectl patch,kubectl delete" check "stale requests carry the genuine pre-rename token; rename changed it" eval 'grep -qx "rv_pre_rename=2" "$T/work/receipt.txt" && grep -qx "rv_post_rename=2-renamed" "$T/work/receipt.txt" && jq -e ".metadata.resourceVersion == \"2\"" "$T/work/stale-update.json" >/dev/null' check "template applied with the long request timeout, then re-applied before any lifecycle mutation" eval '[[ $(grep -c -- "--request-timeout=600s apply -f config/e2e/codertemplate.yaml" "$S/calls.log") -eq 2 ]]' @@ -440,18 +477,18 @@ check "background port-forward and watch stopped" bg_stopped check "every non-streaming kubectl request carries --request-timeout=30s" \ eval '! grep "^kubectl" "$S/calls.log" | grep -v -e "watch=1" -e port-forward | grep -qvE -- "--request-timeout=[0-9]+s "' check "recreate only after the delete job succeeded" eval '[[ $(grep -n include_deleted "$S/calls.log" | head -1 | cut -d: -f1) -lt $(grep -n "create --raw .*ws-e2e-lifecycle-renamed.json" "$S/calls.log" | cut -d: -f1) ]]' -check "receipt: source, run, version, identity, UIDs, 33 passed cases" eval 'grep -q "=== RECEIPT (PASS) ===" "$T/out" && grep -qx "source_sha=0123abc" "$T/work/receipt.txt" && - grep -qx "run_id=42" "$T/work/receipt.txt" && grep -qx "coder_version=v2.37.2+eb69e27" "$T/work/receipt.txt" && grep -qx "uid1=uid-3" "$T/work/receipt.txt" && [[ $(grep -c "= passed$" "$T/work/receipt.txt") -eq 33 ]]' -check "remaining phases: durations recorded; dry run, restart, mid-run, and TTL calls as planned" eval 'grep -Eqx "phase_seconds=agent-failure=[0-9]+s bad-parameter=[0-9]+s restart=[0-9]+s delete-mid-run=[0-9]+s immutability-and-dry-run=[0-9]+s ttl=[0-9]+s ai-provider=[0-9]+s ai-rotation=[0-9]+s ai-refusals=[0-9]+s ai-kill-switch=[0-9]+s ai-delete=[0-9]+s" "$T/work/receipt.txt" && +check "receipt: source, run, version, identity, UIDs, 39 passed cases" eval 'grep -q "=== RECEIPT (PASS) ===" "$T/out" && grep -qx "source_sha=0123abc" "$T/work/receipt.txt" && + grep -qx "run_id=42" "$T/work/receipt.txt" && grep -qx "coder_version=v2.37.2+eb69e27" "$T/work/receipt.txt" && grep -qx "uid1=uid-3" "$T/work/receipt.txt" && [[ $(grep -c "= passed$" "$T/work/receipt.txt") -eq 39 ]]' +check "remaining phases: durations recorded; dry run, restart, mid-run, and TTL calls as planned" eval 'grep -Eqx "phase_seconds=agent-failure=[0-9]+s bad-parameter=[0-9]+s restart=[0-9]+s delete-mid-run=[0-9]+s immutability-and-dry-run=[0-9]+s ttl=[0-9]+s ai-provider=[0-9]+s ai-rotation=[0-9]+s ai-refusals=[0-9]+s ai-kill-switch=[0-9]+s ai-delete=[0-9]+s gw-gate=[0-9]+s gw-not-allowed=[0-9]+s gw-license-required=[0-9]+s gw-secret-not-owned=[0-9]+s gw-leak-check=[0-9]+s gw-delete=[0-9]+s" "$T/work/receipt.txt" && grep -q "create --dry-run=server -f .*tt-e2e-dry-run.json" "$S/calls.log" && grep -q "FROM workspaces WHERE name = .ktt-d0d0d0d0d0d0d0d0d0d0d0d0d0d0." "$S/calls.log" && grep -q "coder-system delete pod coder-k8s-new" "$S/calls.log" && grep -q "workspaces/ttws-e2e-midrun?include_deleted=true" "$S/calls.log" && jq -e ".spec.ttlSecondsAfterFinished == 0" "$T/work/tt-e2e-ttl.json" >/dev/null && jq -e ".spec.parameters == [{name: \"fail\", value: \"notabool\"}]" "$T/work/tt-e2e-bad-param.json" >/dev/null' check "TemplateTest receipt: three durations, tester key counts, namespace deletion time" eval 'grep -Eqx "template_test_seconds=[0-9]+s [0-9]+s [0-9]+s" "$T/work/receipt.txt" && grep -qx "tester_api_keys_before=2" "$T/work/receipt.txt" && grep -qx "tester_api_keys_after=5" "$T/work/receipt.txt" && grep -Eqx "namespace_delete_seconds=[0-9]+" "$T/work/receipt.txt" && out_has "tester api_keys: before=2 after=5 delta=3"' -check "psql only counts rows: operator and tester api_keys, the first pass test's workspace name" eval '[[ $(grep -c "^kubectl .* exec " "$S/calls.log") -eq 9 ]] && +check "psql only counts rows: operator and tester api_keys, the first pass test's workspace name, ai_gateway_keys" eval '[[ $(grep -c "^kubectl .* exec " "$S/calls.log") -eq 14 ]] && ! grep "^kubectl .* exec " "$S/calls.log" | grep -qv "SELECT count(\*) FROM" && grep -q "FROM api_keys WHERE user_id = .user-tester." "$S/calls.log" && - grep -q "FROM workspaces WHERE name = .ktt-e2e-pass-1." "$S/calls.log"' + grep -q "FROM workspaces WHERE name = .ktt-e2e-pass-1." "$S/calls.log" && [[ $(grep -c "FROM ai_gateway_keys$" "$S/calls.log") -eq 5 ]]' check "owner patch names the tester; the namespace test waits 120 s for its agent" eval 'grep -qF "\"ownerUserID\":\"user-tester\"" "$S/calls.log" && jq -e ".spec.parameters == [{name: \"startup_delay\", value: \"120\"}] and .spec.template == \"coder.e2e-agent\"" "$T/work/tt-e2e-ns-delete.json" >/dev/null && jq -e ".spec.version.active and (.spec | has(\"parameters\") | not)" "$T/work/tt-e2e-pass-1.json" >/dev/null' @@ -470,12 +507,19 @@ check "agent workspace: tester-owned, from the agent template, ready time logged jq -e ".metadata.name == \"coder.e2e-tester.e2e-agent\" and .spec.templateName == \"e2e-agent\"" "$T/work/ws-e2e-agent.json" >/dev/null && out_has "agent time-to-ready: " && out_has "agents: main=connected/ready" && grep -q "^agent_ready_seconds=[0-9]" "$T/work/receipt.txt" && grep -q "workspaces/uid-4?include_deleted=true" "$S/calls.log"' -check "AI provider keys: three, never printed or in argv, files removed" eval '[[ $(grep -c . "$S/ai-keys.txt") == 3 ]] && +check "AI provider keys and the AI Gateway Secret value: four, never printed or in argv, files removed" eval '[[ $(grep -c . "$S/ai-keys.txt") == 4 ]] && ! grep -qFf "$S/ai-keys.txt" "$T/out" "$S/calls.log" "$T/work/receipt.txt" && ! compgen -G "$T/work/ai-key-*" && ! compgen -G "$T/work/secret-*.json"' check "AI provider phases: each refusal reason seen; e2e-taken kept in Coder; e2e-openai deleted; e2e-ns-keep released" eval 'out_has "AI provider e2e-p3-unlabeled: False CredentialNotOptedIn" && out_has "AI provider e2e-p3-refused: False CredentialRefused" && out_has "AI provider e2e-p3-origin: False CredentialDestinationNotAllowed" && out_has "AI provider e2e-taken: False NameConflict" && out_has "AI provider e2e-openai: False DisabledExternally" && out_has "AI provider e2e-ns-keep: False ControlPlaneGone" && - [[ -f $S/aip/e2e-taken.json && ! -f $S/aip/e2e-openai.json ]] && jq -e ".spec.apiKeySecretRef == {name: \"op-token\", key: \"token\"}" "$T/work/ai-e2e-p3-refused.json" >/dev/null' + [[ -f $S/aip/e2e-taken.json && ! -f $S/aip/e2e-openai.json ]] && jq -e ".spec.apiKeySecretRef == {name: \"op-token\", key: \"token\"}" "$T/work/ai-e2e-p3-refused.json" >/dev/null && + grep -q "curl -sS --max-time 30 -o .*/coder-raw.json .*/api/v2/ai/providers/aip-e2e-openai$" "$S/calls.log"' +check "AI Gateway key phases: NotAllowed, then LicenseRequired after the opt-in, SecretNotOwned; both objects deleted; a 43-character Secret value" \ + eval 'out_has "AI Gateway key e2e-gw-off: True NotAllowed" && out_has "AI Gateway key e2e-gw-off: True LicenseRequired" && + out_has "AI Gateway key e2e-gw-hijack: True SecretNotOwned" && grep -qF "{\"spec\":{\"aiGateway\":{\"allowKeys\":true}}}" "$S/calls.log" && + [[ ! -f $S/gw/e2e-gw-off.json && ! -f $S/gw/e2e-gw-hijack.json && $(tail -n 1 "$S/ai-keys.txt" | tr -d "\n" | wc -c) == 43 ]] && + jq -e ".spec == {controlPlaneRef: {name: \"coder\"}, secretName: \"e2e-gw-hijack\"}" "$T/work/gw-e2e-gw-hijack.json" >/dev/null && + grep -q "curl -sS --max-time 30 -o .*/coder-raw.json .*/api/v2/ai-gateway/keys$" "$S/calls.log"' echo "TEST image-mismatch: serving image differs from built image" run_scenario image-mismatch SERVING_ID="$OTHER"; summary @@ -665,9 +709,9 @@ tt-failed|template test e2e-pass-1 failed: Failed AgentStartError|$AGENT,kubectl ws-rows-2|expected exactly one workspaces row named ktt-e2e-pass-1 (deleted rows included), found 2|$TESTS coder-rolls|setting templateTests.ownerUserID rolled deploy/coder|$TESTS keys-after-fails|cannot count the api_keys rows of the tester after the tests|$PHASED -coder-stays-up|timed out after 3s waiting for: deploy/coder without pods|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch -ns-delete-stuck|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete -ns-stays-terminating|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete +coder-stays-up|timed out after 3s waiting for: deploy/coder without pods|$GW,kubectl create,kubectl apply,kubectl patch +ns-delete-stuck|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$GW,kubectl create,kubectl apply,kubectl patch,kubectl delete +ns-stays-terminating|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$GW,kubectl create,kubectl apply,kubectl patch,kubectl delete fail-ignored|template test e2e-agent-fail ended Succeeded Succeeded deleted=True/Deleted, want Failed AgentStartError|$TESTS,kubectl create badparam-created|template test e2e-bad-param: expected 0 workspaces rows named ktt-e2e-bad-param (deleted rows included), found 1|$TESTS,kubectl create,kubectl create restart-failed|template test e2e-restart failed: Failed AgentStartError|$TESTS,kubectl create,kubectl create,kubectl create,kubectl delete @@ -693,7 +737,24 @@ ai-wrong-reason|AI provider e2e-p3-origin stalled with False CredentialMalformed ai-conflict-writes|a refused CoderAIProvider wrote to Coder|$PHASED,$P3M ai-kill-undone|the controller undid the disable of provider e2e-openai|$PHASED,$P3D,curl PATCH,kubectl annotate ai-delete-kept|provider e2e-openai (aip-e2e-openai) still exists in Coder after its object was deleted|$PHASED,$AIP -ai-ns-stuck|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$PHASED,$AIP,kubectl create,kubectl apply,kubectl patch,kubectl delete +ai-delete-foreign-404|GET provider aip-e2e-openai after the delete answered HTTP 404 without Coder's own 404 message: {"message":"Route not found."}|$PHASED,$AIP +ai-ns-stuck|timed out after 2s waiting for: test, provider, control plane, and namespace coder deleted|$GW,kubectl create,kubectl apply,kubectl patch,kubectl delete +ai-ns-no-release|the operator log has no ControlPlaneGone release of AI provider e2e-ns-keep|$GW,kubectl create,kubectl apply,kubectl patch,kubectl delete +CASES + +# name|expected message|mutations: each CoderAIGatewayKey phase failure (plan 9) stops before the next mutation. +while IFS='|' read -r name msg muts; do + echo "TEST $name (CoderAIGatewayKey phases)" + run_scenario "$name"