From 66a4e9fa264405ee1aec0744fe9f69c2af34fa35 Mon Sep 17 00:00:00 2001
From: Erik Weathers <1111441+erikdw@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:40:56 -0700
Subject: [PATCH 1/3] Add AWS EKS autoscaling via Prometheus and
 prometheus-adapter.

Stack on merged GKE autoscaling: scrapes all API pools and feeds
per-pool HPAs through custom.metrics with the same event-loop targets.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 braintrust/README.md                          |   6 +
 braintrust/templates/_helpers.tpl             |  11 +-
 .../templates/api-autoscaling-metric.yaml     |   2 +
 .../api-autoscaling-prometheus-adapter.yaml   | 159 ++++++++++++++++++
 .../templates/api-autoscaling-prometheus.yaml | 123 ++++++++++++++
 braintrust/templates/api-hpa.yaml             |  17 ++
 braintrust/tests/api-autoscaling_test.yaml    | 143 +++++++++++++++-
 braintrust/values.yaml                        |  32 +++-
 8 files changed, 483 insertions(+), 10 deletions(-)
 create mode 100644 braintrust/templates/api-autoscaling-prometheus-adapter.yaml
 create mode 100644 braintrust/templates/api-autoscaling-prometheus.yaml

diff --git a/braintrust/README.md b/braintrust/README.md
index 435347c..91cd217 100644
--- a/braintrust/README.md
+++ b/braintrust/README.md
@@ -243,6 +243,12 @@ api:
 
 When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation.<pool>.autoscaling`.
 
+## EKS API Autoscaling
+
+On AWS (`cloud: aws`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`).
+
+Requires API image **v2.9.0+**. With workload isolation enabled, Prometheus scrapes every pool (`api.name`, ingest, and background).
+
 ## API workload isolation
 
 `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest`
diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl
index c5b85ab..c3ff105 100644
--- a/braintrust/templates/_helpers.tpl
+++ b/braintrust/templates/_helpers.tpl
@@ -147,15 +147,18 @@ http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{
 {{- end -}}
 
 {{/*
-Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD).
+Validate API autoscaling prerequisites.
+GKE requires AutoscalingMetric (autoscaling.gke.io/v1beta1).
+EKS uses in-chart Prometheus + prometheus-adapter (no GKE CRD).
 */}}
 {{- define "braintrust.apiAutoscaling.validate" -}}
-{{- if ne .Values.cloud "google" }}
-{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }}
-{{- end }}
+{{- if eq .Values.cloud "google" }}
 {{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }}
 {{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }}
 {{- end }}
+{{- else if ne .Values.cloud "aws" }}
+{{- fail "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" }}
+{{- end }}
 {{- end -}}
 
 {{/*
diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml
index f04245f..16f5694 100644
--- a/braintrust/templates/api-autoscaling-metric.yaml
+++ b/braintrust/templates/api-autoscaling-metric.yaml
@@ -1,4 +1,5 @@
 {{- $root := . -}}
+{{- if eq $root.Values.cloud "google" -}}
 {{- $pools := include "braintrust.apiPools" . | fromYamlArray -}}
 {{- $validated := false -}}
 {{- $rendered := 0 -}}
@@ -51,3 +52,4 @@ spec:
 {{- $rendered = add1 $rendered -}}
 {{- end -}}
 {{- end }}
+{{- end }}
diff --git a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml
new file mode 100644
index 0000000..ca2d137
--- /dev/null
+++ b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml
@@ -0,0 +1,159 @@
+{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces", "pods", "services"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+data:
+  config.yaml: |
+    rules:
+      - seriesQuery: 'braintrust_api_event_loop_delay_mean_seconds{namespace!="",pod!=""}'
+        resources:
+          overrides:
+            namespace:
+              resource: namespace
+            pod:
+              resource: pod
+        name:
+          matches: "^braintrust_api_(.*)"
+          as: "$1"
+        metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+      - seriesQuery: 'braintrust_api_event_loop_utilization_ratio{namespace!="",pod!=""}'
+        resources:
+          overrides:
+            namespace:
+              resource: namespace
+            pod:
+              resource: pod
+        name:
+          matches: "^braintrust_api_(.*)"
+          as: "$1"
+        metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  selector:
+    app: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  ports:
+    - name: https
+      port: 443
+      targetPort: 6443
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  service:
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+    port: 443
+  group: custom.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    spec:
+      serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+      containers:
+        - name: prometheus-adapter
+          image: "{{ .Values.api.autoscaling.adapter.image.repository }}:{{ .Values.api.autoscaling.adapter.image.tag }}"
+          imagePullPolicy: {{ .Values.api.autoscaling.adapter.image.pullPolicy }}
+          args:
+            - --secure-port=6443
+            - --cert-dir=/tmp/cert
+            - --prometheus-url=http://{{ .Values.api.name }}-autoscaling-prometheus.{{ include "braintrust.namespace" . }}.svc:9090
+            - --metrics-relist-interval={{ .Values.api.autoscaling.prometheus.scrapeInterval }}
+            - --config=/etc/adapter/config.yaml
+            - --v=2
+          ports:
+            - containerPort: 6443
+              name: https
+          resources:
+            {{- toYaml .Values.api.autoscaling.adapter.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/adapter
+            - name: tmp
+              mountPath: /tmp/cert
+      volumes:
+        - name: config
+          configMap:
+            name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+        - name: tmp
+          emptyDir: {}
+{{- end }}
diff --git a/braintrust/templates/api-autoscaling-prometheus.yaml b/braintrust/templates/api-autoscaling-prometheus.yaml
new file mode 100644
index 0000000..be19129
--- /dev/null
+++ b/braintrust/templates/api-autoscaling-prometheus.yaml
@@ -0,0 +1,123 @@
+{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }}
+{{- $scrapeApps := list .Values.api.name -}}
+{{- if .Values.api.workloadIsolation.enabled -}}
+{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.ingest.name -}}
+{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.background.name -}}
+{{- end -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }}
+      evaluation_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }}
+
+    scrape_configs:
+      - job_name: braintrust-api-autoscaling
+        kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              names:
+                - {{ include "braintrust.namespace" . }}
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_label_app]
+            regex: {{ join "|" $scrapeApps }}
+            action: keep
+          - source_labels: [__meta_kubernetes_pod_ip]
+            target_label: __address__
+            replacement: $1:{{ .Values.api.healthServer.port }}
+          - source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+        metrics_path: {{ .Values.api.autoscaling.metricsPath }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  selector:
+    app: {{ .Values.api.name }}-autoscaling-prometheus
+  ports:
+    - name: http
+      port: 9090
+      targetPort: 9090
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.api.name }}-autoscaling-prometheus
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.api.name }}-autoscaling-prometheus
+    spec:
+      serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus
+      containers:
+        - name: prometheus
+          image: "{{ .Values.api.autoscaling.prometheus.image.repository }}:{{ .Values.api.autoscaling.prometheus.image.tag }}"
+          imagePullPolicy: {{ .Values.api.autoscaling.prometheus.image.pullPolicy }}
+          args:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention.time={{ .Values.api.autoscaling.prometheus.retention }}
+            - --web.enable-lifecycle
+          ports:
+            - containerPort: 9090
+              name: http
+          resources:
+            {{- toYaml .Values.api.autoscaling.prometheus.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/prometheus
+            - name: data
+              mountPath: /prometheus
+      volumes:
+        - name: config
+          configMap:
+            name: {{ .Values.api.name }}-autoscaling-prometheus
+        - name: data
+          emptyDir: {}
+{{- end }}
diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml
index 4b2d97e..10991f1 100644
--- a/braintrust/templates/api-hpa.yaml
+++ b/braintrust/templates/api-hpa.yaml
@@ -47,6 +47,7 @@ spec:
         target:
           type: Utilization
           averageUtilization: {{ $api.autoscaling.cpu.targetAverageUtilization }}
+    {{- if eq $root.Values.cloud "google" }}
     - type: Pods
       pods:
         metric:
@@ -61,6 +62,22 @@ spec:
         target:
           type: AverageValue
           averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }}
+    {{- else if eq $root.Values.cloud "aws" }}
+    - type: Pods
+      pods:
+        metric:
+          name: event_loop_utilization_ratio
+        target:
+          type: AverageValue
+          averageValue: {{ $api.autoscaling.eventLoopUtilization.targetAverageValue | quote }}
+    - type: Pods
+      pods:
+        metric:
+          name: event_loop_delay_mean_seconds
+        target:
+          type: AverageValue
+          averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }}
+    {{- end }}
   {{- with $api.autoscaling.behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}
diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml
index cb22f4c..01dc574 100644
--- a/braintrust/tests/api-autoscaling_test.yaml
+++ b/braintrust/tests/api-autoscaling_test.yaml
@@ -2,6 +2,8 @@ suite: test API autoscaling templates
 templates:
   - api-autoscaling-metric.yaml
   - api-hpa.yaml
+  - api-autoscaling-prometheus.yaml
+  - api-autoscaling-prometheus-adapter.yaml
   - api-deployment.yaml
   - api-configmap.yaml
 capabilities:
@@ -34,18 +36,18 @@ tests:
       - hasDocuments:
           count: 0
 
-  - it: should fail when autoscaling is enabled on a non-google cloud
+  - it: should fail when autoscaling is enabled on an unsupported cloud
     template: api-hpa.yaml
     values:
       - __fixtures__/base-values.yaml
-      - __fixtures__/aws-values.yaml
+      - __fixtures__/azure-values.yaml
     set:
       api.autoscaling.enabled: true
     release:
       namespace: "braintrust"
     asserts:
       - failedTemplate:
-          errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE)"
+          errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)"
 
   - it: should render AutoscalingMetric for GKE with event-loop gauges
     template: api-autoscaling-metric.yaml
@@ -435,3 +437,138 @@ tests:
       - hasDocuments:
           count: 0
           filterAware: true
+
+  - it: should not render GKE AutoscalingMetric on AWS
+    template: api-autoscaling-metric.yaml
+    values:
+      - __fixtures__/base-values.yaml
+      - __fixtures__/aws-values.yaml
+    set:
+      api.autoscaling.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should not render Prometheus stack on GKE
+    template: api-autoscaling-prometheus.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      cloud: google
+      api.autoscaling.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should render HPA on AWS with prometheus-adapter metric names
+    template: api-hpa.yaml
+    values:
+      - __fixtures__/base-values.yaml
+      - __fixtures__/aws-values.yaml
+    set:
+      api.autoscaling.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - isKind:
+          of: HorizontalPodAutoscaler
+      - equal:
+          path: spec.metrics[0].type
+          value: ContainerResource
+      - equal:
+          path: spec.metrics[0].containerResource.container
+          value: api
+      - equal:
+          path: spec.metrics[1].pods.metric.name
+          value: event_loop_utilization_ratio
+      - equal:
+          path: spec.metrics[1].pods.target.averageValue
+          value: "0.4"
+      - equal:
+          path: spec.metrics[2].pods.metric.name
+          value: event_loop_delay_mean_seconds
+      - equal:
+          path: spec.metrics[2].pods.target.averageValue
+          value: "0.05"
+
+  - it: should scrape health server metrics on AWS Prometheus
+    template: api-autoscaling-prometheus.yaml
+    documentSelector:
+      path: kind
+      value: ConfigMap
+    values:
+      - __fixtures__/base-values.yaml
+      - __fixtures__/aws-values.yaml
+    set:
+      api.autoscaling.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - matchRegex:
+          path: data["prometheus.yml"]
+          pattern: ":8001"
+      - matchRegex:
+          path: data["prometheus.yml"]
+          pattern: "metrics_path: /metrics"
+      - matchRegex:
+          path: data["prometheus.yml"]
+          pattern: "regex: braintrust-api"
+
+  - it: should scrape all API pools on AWS when workload isolation is enabled
+    template: api-autoscaling-prometheus.yaml
+    documentSelector:
+      path: kind
+      value: ConfigMap
+    values:
+      - __fixtures__/base-values.yaml
+      - __fixtures__/aws-values.yaml
+    set:
+      api.autoscaling.enabled: true
+      api.workloadIsolation.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - matchRegex:
+          path: data["prometheus.yml"]
+          pattern: "braintrust-api-ingest"
+      - matchRegex:
+          path: data["prometheus.yml"]
+          pattern: "braintrust-api-background"
+
+  - it: should render three HPAs on AWS when workload isolation is enabled
+    template: api-hpa.yaml
+    values:
+      - __fixtures__/base-values.yaml
+      - __fixtures__/aws-values.yaml
+    set:
+      api.autoscaling.enabled: true
+      api.workloadIsolation.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - hasDocuments:
+          count: 3
+
+  - it: should render prometheus-adapter rules for event-loop gauges on AWS
+    template: api-autoscaling-prometheus-adapter.yaml
+    documentSelector:
+      path: kind
+      value: ConfigMap
+    values:
+      - __fixtures__/base-values.yaml
+      - __fixtures__/aws-values.yaml
+    set:
+      api.autoscaling.enabled: true
+    release:
+      namespace: "braintrust"
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: "braintrust_api_event_loop_delay_mean_seconds"
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: "braintrust_api_event_loop_utilization_ratio"
diff --git a/braintrust/values.yaml b/braintrust/values.yaml
index d91754f..3785374 100644
--- a/braintrust/values.yaml
+++ b/braintrust/values.yaml
@@ -133,19 +133,20 @@ api:
     hpa: {}
     autoscalingMetric: {}
   replicas: 4
-  # Autoscale API pools on CPU and Node.js event-loop metrics via AutoscalingMetric + HPA.
+  # Autoscale API pools on CPU and Node.js event-loop metrics.
   # Applies to every rendered API pool (default, and ingest/background when
   # workloadIsolation is enabled). Pool overrides can set minReplicas/maxReplicas
   # (or disable) under api.workloadIsolation.<pool>.autoscaling.
-  # The GKE feature for custom autoscaling metrics is in Preview (Pre-GA). It requires:
+  # cloud=google (GKE): AutoscalingMetric CRD + HPA. Preview (Pre-GA); requires:
   # - Braintrust API / data plane v2.9.0+ (Prometheus /metrics on the API health server)
   # - GKE 1.35.1-gke.1396000 or later
   # - the Performance HPA profile and Autoscaling API
   # - roles/autoscaling.metricsWriter on all node service accounts
   # - the Autoscaling API in the service perimeter when using VPC Service Controls
   # Helm install/upgrade fails if autoscaling.gke.io/v1beta1 is not on the cluster.
-  # When enabled for a pool, that pool's replicas setting is ignored.
   # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling
+  # cloud=aws (EKS): in-chart Prometheus + prometheus-adapter + HPA (custom.metrics).
+  # When enabled for a pool, that pool's replicas setting is ignored.
   autoscaling:
     enabled: false
     minReplicas: 4
@@ -166,6 +167,31 @@ api:
         stabilizationWindowSeconds: 300
       scaleUp:
         stabilizationWindowSeconds: 60
+    # Used when cloud=aws (Prometheus + prometheus-adapter).
+    prometheus:
+      image:
+        repository: prom/prometheus
+        tag: v3.8.0
+        pullPolicy: IfNotPresent
+      retention: 1h
+      scrapeInterval: 15s
+      resources:
+        requests:
+          cpu: 100m
+          memory: 256Mi
+        limits:
+          memory: 512Mi
+    adapter:
+      image:
+        repository: registry.k8s.io/prometheus-adapter/prometheus-adapter
+        tag: v0.12.0
+        pullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          memory: 256Mi
   # Optional fixed-capacity workload isolation. This creates dedicated API
   # Deployments and Services for ingestion and background work while retaining
   # api.name as the default pool for all remaining traffic. Public ingress must

From 3e8690f7c305015ae197db82391e421bb8124805 Mon Sep 17 00:00:00 2001
From: Erik Weathers <1111441+erikdw@users.noreply.github.com>
Date: Wed, 28 Oct 2026 13:09:52 -0700
Subject: [PATCH 2/3] docs: add EKS/AKS autoscaling metrics handoff

Capture why we chose Prometheus + prometheus-adapter vs a
GKE-style AutoscalingMetric controller, and reanalyze for 2026
(KEDA / adapter deprecation trajectory).

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 HANDOFF.md | 166 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)
 create mode 100644 HANDOFF.md

diff --git a/HANDOFF.md b/HANDOFF.md
new file mode 100644
index 0000000..b15994f
--- /dev/null
+++ b/HANDOFF.md
@@ -0,0 +1,166 @@
+# Handoff: EKS / AKS API autoscaling metrics path
+
+**Audience:** Yash (and anyone finishing [#94](https://github.com/braintrustdata/helm/pull/94) / [#96](https://github.com/braintrustdata/helm/pull/96))  
+**Status:** Rebased onto `main` after GKE autoscaling merged ([#95](https://github.com/braintrustdata/helm/pull/95)). Chart unit tests pass. EKS sandbox was soaked earlier; re-soak before ship. AKS not soaked.  
+**Date:** 2026-10-28
+
+This note explains **why we chose in-chart Prometheus + prometheus-adapter for EKS/AKS**, how that relates to GKE’s `AutoscalingMetric`, and whether that choice still holds after re-checking the ecosystem.
+
+---
+
+## What we are trying to do
+
+Match **ECS api-ts autoscaling signals** on Kubernetes:
+
+| Signal | Target (prom-canonical) |
+| --- | --- |
+| CPU | 50% utilization on the `api` container (`ContainerResource`) |
+| Event-loop utilization | average `0.4` (ratio 0–1) |
+| Event-loop delay mean | average `0.05` seconds |
+
+Product constraints locked earlier:
+
+1. **Braintrust owns** the autoscaling metrics path (not “customer already has Prometheus”).
+2. **Same app contract everywhere:** API health `/metrics` (`ENABLE_PROMETHEUS_METRICS`, data plane **v2.9.0+**).
+3. **HPA scales pods**; node scaling (Karpenter / NAP) is separate.
+4. **api-ts only** for v1 (brainstore HPA out of scope).
+5. **Isolated from** Datadog / Pulse / `controlPlaneTelemetry`.
+
+GKE (#95) uses Google’s managed **`AutoscalingMetric`** CRD: the control plane scrapes each pool’s `/metrics` and feeds HPA. EKS and AKS have **no equivalent managed CRD**.
+
+---
+
+## Architecture we shipped on EKS / AKS
+
+```
+api pods (:8001/metrics)
+        │ scrape
+        ▼
+in-chart Prometheus (short retention)
+        │ query
+        ▼
+prometheus-adapter  ──APIService──▶  custom.metrics.k8s.io
+        │
+        ▼
+HorizontalPodAutoscaler (Pods metrics: event_loop_*)
+```
+
+- Templates: `api-autoscaling-prometheus.yaml`, `api-autoscaling-prometheus-adapter.yaml`, cloud-branched `api-hpa.yaml`.
+- GKE keeps `AutoscalingMetric`; EKS/AKS skip that CRD and skip the GKE API discovery fail in `braintrust.apiAutoscaling.validate`.
+- With workload isolation, Prometheus scrapes **all** pool apps; each pool still gets its own HPA (or fixed replicas if that pool disables autoscaling).
+
+PRs:
+
+| PR | Branch | Base | Role |
+| --- | --- | --- | --- |
+| [#94](https://github.com/braintrustdata/helm/pull/94) | `erikdw/api-autoscaling-aws-prom` | `main` | EKS (`cloud=aws`) |
+| [#96](https://github.com/braintrustdata/helm/pull/96) | `erikdw/api-autoscaling-aks` | #94 | AKS (`cloud=azure`) |
+
+Merge order: **#94 then #96**.
+
+---
+
+## Original rationale for Prometheus + prometheus-adapter
+
+### Why not “just HPA”?
+
+HPA only natively understands **resource** metrics (`metrics.k8s.io` via metrics-server). Event-loop gauges are **application metrics**. Something must implement `custom.metrics.k8s.io` (or `external.metrics.k8s.io`) and register an APIService, or HPA cannot see them.
+
+### Why Prometheus in the chart?
+
+We need a scrape + store of the Prometheus text format already emitted by api-ts. Options were:
+
+| Option | Why we rejected or deferred |
+| --- | --- |
+| Customer BYO Prometheus | Violates “Braintrust owns autoscaling metrics.” |
+| CloudWatch push (ECS path) | AWS-only; different contract from GKE `/metrics`; needs a CloudWatch→HPA adapter; duplicates ECS instrumentation for K8s. |
+| Amazon Managed Prometheus / Azure Monitor | Still need a bridge to HPA; cloud-specific IAM and ops; harder to keep one path for EKS **and** AKS. |
+| Sidecar scraper per API pod | More moving parts on the hot path; still need a metrics API for HPA. |
+
+A **small in-chart Prometheus** (1h retention, scrape only API pool pods) is the simplest Braintrust-owned store that matches the app’s existing exposition format.
+
+### Why prometheus-adapter (vs inventing a GKE-like controller)?
+
+GKE `AutoscalingMetric` is roughly: **declare which pod gauges to scrape → managed controller publishes them for HPA**. On EKS there is no first-party twin.
+
+**Building our own “AutoscalingMetric for EKS”** would mean owning:
+
+1. A CRD and validation UX.
+2. A scraper (or pull agent) hitting every API pod `/metrics`.
+3. An aggregated API server implementing `custom.metrics.k8s.io` (auth delegation, discovery, watch semantics HPA expects).
+4. Multi-cloud packaging, upgrades, RBAC, failure modes, and support for self-hosted customers.
+
+That is a **metrics-adapter product**, not a thin Helm feature. Cost and risk dwarf the benefit when mature adapters already do (3)+(part of 2) and Prometheus already does scrape/store.
+
+**prometheus-adapter** was the standard kubernetes-sigs bridge: PromQL series → `custom.metrics.k8s.io` → native `autoscaling/v2` HPA with `type: Pods`. It kept:
+
+- The same **HPA object shape** as GKE (CPU + two Pods metrics).
+- The same **metric semantics** (per-pod averages of ELU / delay).
+- A stack we can also turn on for **AKS** with a one-line cloud gate (#96).
+
+Closest “GKE-like open source” alternatives (e.g. Zalando/Demonware `kube-metrics-adapter`) still leave us maintaining a less-common adapter and do not remove the need for scrape + APIService ownership.
+
+---
+
+## Reanalysis (2026-10): is prometheus-adapter still the right move?
+
+**Short answer:** It remains a **reasonable path to finish and soak #94/#96**, and it is still better than writing a GKE-style controller. It is **no longer the clear long-term default** — plan a near-term evaluation of **KEDA** before treating Prom+adapter as permanent.
+
+### What changed
+
+- SIG Instrumentation has an **open proposal to deprecate prometheus-adapter** and steer users to **KEDA** ([kubernetes-sigs/prometheus-adapter#701](https://github.com/kubernetes-sigs/prometheus-adapter/issues/701), triage/accepted; deprecation not formally announced as of late Oct 2026, chart still publishing).
+- AWS publishes guidance for **KEDA on EKS** for custom/event-driven autoscaling.
+- Community docs increasingly warn: do not run KEDA and prometheus-adapter both as owners of `external.metrics.k8s.io`.
+
+### Comparison for *our* use case
+
+We need: average of two **pod gauges** + CPU `ContainerResource`, minReplicas ≥ 1 (no scale-to-zero), Braintrust-owned scrape, portable EKS+AKS.
+
+| Approach | Fit | Pros | Cons |
+| --- | --- | --- | --- |
+| **Prom + prometheus-adapter (current PRs)** | Good short-term | Native HPA; `custom.metrics` Pods metrics; already implemented + unit-tested; prior EKS soak; AKS reuse | Upstream deprecation trajectory; APIService conflict if cluster already has an adapter; we run a second Prom |
+| **Prom + KEDA (Prometheus trigger / ScaledObject)** | Strong medium-term | Actively maintained; AWS-aligned; one operator story; avoids PA deprecation | Different UX (`ScaledObject` vs hand-written HPA); typically `external.metrics`; need to prove ELU/delay average semantics match ECS/GKE; still need a Prom (or AMP) |
+| **Homegrown AutoscalingMetric-like controller** | Poor | Closest DX to GKE CRD | High build/ops cost; reinvent metrics API; no customer demand that outweighs adapters |
+| **CloudWatch metrics adapter** | Poor for unified K8s | Reuses ECS push mental model | AWS-only; diverges from GKE `/metrics`; extra IAM |
+| **BYO customer Prom/adapter** | Poor vs product goal | Less chart bulk | Breaks “turn on autoscaling” for self-hosted |
+
+### Recommendation for Yash
+
+1. **Ship / finish #94 on Prom+adapter** if the goal is parity and soak soon — do not block on rewriting to KEDA mid-flight unless review insists.
+2. **Explicit follow-up ticket:** spike KEDA replacing the adapter (keep in-chart or AMP Prometheus; keep `/metrics` contract; match replica behavior and metric targets). Prefer one path for EKS+AKS.
+3. **Do not** invest in a Braintrust-owned GKE `AutoscalingMetric` clone unless Google’s managed path becomes unavailable on GKE *and* no adapter/KEDA path meets support needs.
+4. Before production: check the target cluster for an existing `v1beta1.custom.metrics.k8s.io` / `external.metrics.k8s.io` APIService — dual adapters break HPA.
+
+---
+
+## Remaining work checklist
+
+- [ ] Re-soak EKS (`api.autoscaling.enabled=true`): Prom targets up on `:8001`, HPA has non-empty `event_loop_*`, scale under load.
+- [ ] First AKS soak (#96) with same checks.
+- [ ] Example values file for EKS (and AKS) analogous to `examples/google-api-isolation-autoscaling/`.
+- [ ] Document APIService conflict / single-adapter assumption in README.
+- [ ] Decide PA → KEDA migration timing (before or after first customer GA).
+- [ ] Confirm image pins (`prom/prometheus`, `prometheus-adapter`) and resource requests for small sandboxes vs prod.
+
+### Local commands
+
+```bash
+cd /path/to/helm
+./test.sh
+```
+
+### Unrelated local stash
+
+Before this rebase, unrelated WIP was stashed on the old AKS checkout:
+
+`stash@{0}: unrelated-wip-before-autoscaling-rebase` (configmap / helpers bits). Not part of these PRs.
+
+---
+
+## Decision summary
+
+| Question | Answer |
+| --- | --- |
+| Why Prom+adapter originally? | Braintrust-owned scrape + standard bridge to HPA custom metrics; same `/metrics` as GKE; portable to AKS; far cheaper than a custom GKE-like controller. |
+| Still right for EKS *today*? | **Yes to complete and validate the open PRs.** Treat as **interim** given PA’s deprecation proposal; spike **KEDA** as the likely durable EKS/AKS backend. |
+| Build our own AutoscalingMetric controller? | **No** — wrong ROI; duplicates adapter + aggregation-layer work Google already runs as a managed Preview on GKE. |

From ea254c8a995bcca258cc8db2758d975c0a5aae86 Mon Sep 17 00:00:00 2001
From: Erik Weathers <1111441+erikdw@users.noreply.github.com>
Date: Wed, 28 Oct 2026 13:10:09 -0700
Subject: [PATCH 3/3] docs: remove HANDOFF.md from chart PR

Handoff writeup belongs in Linear, not the helm PR.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 HANDOFF.md | 166 -----------------------------------------------------
 1 file changed, 166 deletions(-)
 delete mode 100644 HANDOFF.md

diff --git a/HANDOFF.md b/HANDOFF.md
deleted file mode 100644
index b15994f..0000000
--- a/HANDOFF.md
+++ /dev/null
@@ -1,166 +0,0 @@
-# Handoff: EKS / AKS API autoscaling metrics path
-
-**Audience:** Yash (and anyone finishing [#94](https://github.com/braintrustdata/helm/pull/94) / [#96](https://github.com/braintrustdata/helm/pull/96))  
-**Status:** Rebased onto `main` after GKE autoscaling merged ([#95](https://github.com/braintrustdata/helm/pull/95)). Chart unit tests pass. EKS sandbox was soaked earlier; re-soak before ship. AKS not soaked.  
-**Date:** 2026-10-28
-
-This note explains **why we chose in-chart Prometheus + prometheus-adapter for EKS/AKS**, how that relates to GKE’s `AutoscalingMetric`, and whether that choice still holds after re-checking the ecosystem.
-
----
-
-## What we are trying to do
-
-Match **ECS api-ts autoscaling signals** on Kubernetes:
-
-| Signal | Target (prom-canonical) |
-| --- | --- |
-| CPU | 50% utilization on the `api` container (`ContainerResource`) |
-| Event-loop utilization | average `0.4` (ratio 0–1) |
-| Event-loop delay mean | average `0.05` seconds |
-
-Product constraints locked earlier:
-
-1. **Braintrust owns** the autoscaling metrics path (not “customer already has Prometheus”).
-2. **Same app contract everywhere:** API health `/metrics` (`ENABLE_PROMETHEUS_METRICS`, data plane **v2.9.0+**).
-3. **HPA scales pods**; node scaling (Karpenter / NAP) is separate.
-4. **api-ts only** for v1 (brainstore HPA out of scope).
-5. **Isolated from** Datadog / Pulse / `controlPlaneTelemetry`.
-
-GKE (#95) uses Google’s managed **`AutoscalingMetric`** CRD: the control plane scrapes each pool’s `/metrics` and feeds HPA. EKS and AKS have **no equivalent managed CRD**.
-
----
-
-## Architecture we shipped on EKS / AKS
-
-```
-api pods (:8001/metrics)
-        │ scrape
-        ▼
-in-chart Prometheus (short retention)
-        │ query
-        ▼
-prometheus-adapter  ──APIService──▶  custom.metrics.k8s.io
-        │
-        ▼
-HorizontalPodAutoscaler (Pods metrics: event_loop_*)
-```
-
-- Templates: `api-autoscaling-prometheus.yaml`, `api-autoscaling-prometheus-adapter.yaml`, cloud-branched `api-hpa.yaml`.
-- GKE keeps `AutoscalingMetric`; EKS/AKS skip that CRD and skip the GKE API discovery fail in `braintrust.apiAutoscaling.validate`.
-- With workload isolation, Prometheus scrapes **all** pool apps; each pool still gets its own HPA (or fixed replicas if that pool disables autoscaling).
-
-PRs:
-
-| PR | Branch | Base | Role |
-| --- | --- | --- | --- |
-| [#94](https://github.com/braintrustdata/helm/pull/94) | `erikdw/api-autoscaling-aws-prom` | `main` | EKS (`cloud=aws`) |
-| [#96](https://github.com/braintrustdata/helm/pull/96) | `erikdw/api-autoscaling-aks` | #94 | AKS (`cloud=azure`) |
-
-Merge order: **#94 then #96**.
-
----
-
-## Original rationale for Prometheus + prometheus-adapter
-
-### Why not “just HPA”?
-
-HPA only natively understands **resource** metrics (`metrics.k8s.io` via metrics-server). Event-loop gauges are **application metrics**. Something must implement `custom.metrics.k8s.io` (or `external.metrics.k8s.io`) and register an APIService, or HPA cannot see them.
-
-### Why Prometheus in the chart?
-
-We need a scrape + store of the Prometheus text format already emitted by api-ts. Options were:
-
-| Option | Why we rejected or deferred |
-| --- | --- |
-| Customer BYO Prometheus | Violates “Braintrust owns autoscaling metrics.” |
-| CloudWatch push (ECS path) | AWS-only; different contract from GKE `/metrics`; needs a CloudWatch→HPA adapter; duplicates ECS instrumentation for K8s. |
-| Amazon Managed Prometheus / Azure Monitor | Still need a bridge to HPA; cloud-specific IAM and ops; harder to keep one path for EKS **and** AKS. |
-| Sidecar scraper per API pod | More moving parts on the hot path; still need a metrics API for HPA. |
-
-A **small in-chart Prometheus** (1h retention, scrape only API pool pods) is the simplest Braintrust-owned store that matches the app’s existing exposition format.
-
-### Why prometheus-adapter (vs inventing a GKE-like controller)?
-
-GKE `AutoscalingMetric` is roughly: **declare which pod gauges to scrape → managed controller publishes them for HPA**. On EKS there is no first-party twin.
-
-**Building our own “AutoscalingMetric for EKS”** would mean owning:
-
-1. A CRD and validation UX.
-2. A scraper (or pull agent) hitting every API pod `/metrics`.
-3. An aggregated API server implementing `custom.metrics.k8s.io` (auth delegation, discovery, watch semantics HPA expects).
-4. Multi-cloud packaging, upgrades, RBAC, failure modes, and support for self-hosted customers.
-
-That is a **metrics-adapter product**, not a thin Helm feature. Cost and risk dwarf the benefit when mature adapters already do (3)+(part of 2) and Prometheus already does scrape/store.
-
-**prometheus-adapter** was the standard kubernetes-sigs bridge: PromQL series → `custom.metrics.k8s.io` → native `autoscaling/v2` HPA with `type: Pods`. It kept:
-
-- The same **HPA object shape** as GKE (CPU + two Pods metrics).
-- The same **metric semantics** (per-pod averages of ELU / delay).
-- A stack we can also turn on for **AKS** with a one-line cloud gate (#96).
-
-Closest “GKE-like open source” alternatives (e.g. Zalando/Demonware `kube-metrics-adapter`) still leave us maintaining a less-common adapter and do not remove the need for scrape + APIService ownership.
-
----
-
-## Reanalysis (2026-10): is prometheus-adapter still the right move?
-
-**Short answer:** It remains a **reasonable path to finish and soak #94/#96**, and it is still better than writing a GKE-style controller. It is **no longer the clear long-term default** — plan a near-term evaluation of **KEDA** before treating Prom+adapter as permanent.
-
-### What changed
-
-- SIG Instrumentation has an **open proposal to deprecate prometheus-adapter** and steer users to **KEDA** ([kubernetes-sigs/prometheus-adapter#701](https://github.com/kubernetes-sigs/prometheus-adapter/issues/701), triage/accepted; deprecation not formally announced as of late Oct 2026, chart still publishing).
-- AWS publishes guidance for **KEDA on EKS** for custom/event-driven autoscaling.
-- Community docs increasingly warn: do not run KEDA and prometheus-adapter both as owners of `external.metrics.k8s.io`.
-
-### Comparison for *our* use case
-
-We need: average of two **pod gauges** + CPU `ContainerResource`, minReplicas ≥ 1 (no scale-to-zero), Braintrust-owned scrape, portable EKS+AKS.
-
-| Approach | Fit | Pros | Cons |
-| --- | --- | --- | --- |
-| **Prom + prometheus-adapter (current PRs)** | Good short-term | Native HPA; `custom.metrics` Pods metrics; already implemented + unit-tested; prior EKS soak; AKS reuse | Upstream deprecation trajectory; APIService conflict if cluster already has an adapter; we run a second Prom |
-| **Prom + KEDA (Prometheus trigger / ScaledObject)** | Strong medium-term | Actively maintained; AWS-aligned; one operator story; avoids PA deprecation | Different UX (`ScaledObject` vs hand-written HPA); typically `external.metrics`; need to prove ELU/delay average semantics match ECS/GKE; still need a Prom (or AMP) |
-| **Homegrown AutoscalingMetric-like controller** | Poor | Closest DX to GKE CRD | High build/ops cost; reinvent metrics API; no customer demand that outweighs adapters |
-| **CloudWatch metrics adapter** | Poor for unified K8s | Reuses ECS push mental model | AWS-only; diverges from GKE `/metrics`; extra IAM |
-| **BYO customer Prom/adapter** | Poor vs product goal | Less chart bulk | Breaks “turn on autoscaling” for self-hosted |
-
-### Recommendation for Yash
-
-1. **Ship / finish #94 on Prom+adapter** if the goal is parity and soak soon — do not block on rewriting to KEDA mid-flight unless review insists.
-2. **Explicit follow-up ticket:** spike KEDA replacing the adapter (keep in-chart or AMP Prometheus; keep `/metrics` contract; match replica behavior and metric targets). Prefer one path for EKS+AKS.
-3. **Do not** invest in a Braintrust-owned GKE `AutoscalingMetric` clone unless Google’s managed path becomes unavailable on GKE *and* no adapter/KEDA path meets support needs.
-4. Before production: check the target cluster for an existing `v1beta1.custom.metrics.k8s.io` / `external.metrics.k8s.io` APIService — dual adapters break HPA.
-
----
-
-## Remaining work checklist
-
-- [ ] Re-soak EKS (`api.autoscaling.enabled=true`): Prom targets up on `:8001`, HPA has non-empty `event_loop_*`, scale under load.
-- [ ] First AKS soak (#96) with same checks.
-- [ ] Example values file for EKS (and AKS) analogous to `examples/google-api-isolation-autoscaling/`.
-- [ ] Document APIService conflict / single-adapter assumption in README.
-- [ ] Decide PA → KEDA migration timing (before or after first customer GA).
-- [ ] Confirm image pins (`prom/prometheus`, `prometheus-adapter`) and resource requests for small sandboxes vs prod.
-
-### Local commands
-
-```bash
-cd /path/to/helm
-./test.sh
-```
-
-### Unrelated local stash
-
-Before this rebase, unrelated WIP was stashed on the old AKS checkout:
-
-`stash@{0}: unrelated-wip-before-autoscaling-rebase` (configmap / helpers bits). Not part of these PRs.
-
----
-
-## Decision summary
-
-| Question | Answer |
-| --- | --- |
-| Why Prom+adapter originally? | Braintrust-owned scrape + standard bridge to HPA custom metrics; same `/metrics` as GKE; portable to AKS; far cheaper than a custom GKE-like controller. |
-| Still right for EKS *today*? | **Yes to complete and validate the open PRs.** Treat as **interim** given PA’s deprecation proposal; spike **KEDA** as the likely durable EKS/AKS backend. |
-| Build our own AutoscalingMetric controller? | **No** — wrong ROI; duplicates adapter + aggregation-layer work Google already runs as a managed Preview on GKE. |