Erik Weathers (erikdw) · Oct 27, 2026 · Oct 28, 2026 · Oct 28, 2026
diff --git a/braintrust/README.md b/braintrust/README.md
@@ -243,6 +243,12 @@ api:
 
 When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation.<pool>.autoscaling`.
 
+## EKS API Autoscaling
+
+On AWS (`cloud: aws`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`).
+
+Requires API image **v2.9.0+**. With workload isolation enabled, Prometheus scrapes every pool (`api.name`, ingest, and background).
+
 ## API workload isolation
 
 `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest`

diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl
@@ -147,15 +147,18 @@ http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{
 {{- end -}}
 
 {{/*
-Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD).
+Validate API autoscaling prerequisites.
+GKE requires AutoscalingMetric (autoscaling.gke.io/v1beta1).
+EKS uses in-chart Prometheus + prometheus-adapter (no GKE CRD).
 */}}
 {{- define "braintrust.apiAutoscaling.validate" -}}
-{{- if ne .Values.cloud "google" }}
-{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }}
-{{- end }}
+{{- if eq .Values.cloud "google" }}
 {{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }}
 {{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }}
 {{- end }}
+{{- else if ne .Values.cloud "aws" }}
+{{- fail "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" }}
+{{- end }}
 {{- end -}}
 
 {{/*

diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml
@@ -1,4 +1,5 @@
 {{- $root := . -}}
+{{- if eq $root.Values.cloud "google" -}}
 {{- $pools := include "braintrust.apiPools" . | fromYamlArray -}}
 {{- $validated := false -}}
 {{- $rendered := 0 -}}
@@ -51,3 +52,4 @@ spec:
 {{- $rendered = add1 $rendered -}}
 {{- end -}}
 {{- end }}
+{{- end }}
diff --git a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml
@@ -0,0 +1,159 @@
+{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces", "pods", "services"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+data:
+  config.yaml: |
+    rules:
+      - seriesQuery: 'braintrust_api_event_loop_delay_mean_seconds{namespace!="",pod!=""}'
+        resources:
+          overrides:
+            namespace:
+              resource: namespace
+            pod:
+              resource: pod
+        name:
+          matches: "^braintrust_api_(.*)"
+          as: "$1"
+        metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+      - seriesQuery: 'braintrust_api_event_loop_utilization_ratio{namespace!="",pod!=""}'
+        resources:
+          overrides:
+            namespace:
+              resource: namespace
+            pod:
+              resource: pod
+        name:
+          matches: "^braintrust_api_(.*)"
+          as: "$1"
+        metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  selector:
+    app: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  ports:
+    - name: https
+      port: 443
+      targetPort: 6443
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  service:
+    name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    namespace: {{ include "braintrust.namespace" . }}
+    port: 443
+  group: custom.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+    spec:
+      serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+      containers:
+        - name: prometheus-adapter
+          image: "{{ .Values.api.autoscaling.adapter.image.repository }}:{{ .Values.api.autoscaling.adapter.image.tag }}"
+          imagePullPolicy: {{ .Values.api.autoscaling.adapter.image.pullPolicy }}
+          args:
+            - --secure-port=6443
+            - --cert-dir=/tmp/cert
+            - --prometheus-url=http://{{ .Values.api.name }}-autoscaling-prometheus.{{ include "braintrust.namespace" . }}.svc:9090
+            - --metrics-relist-interval={{ .Values.api.autoscaling.prometheus.scrapeInterval }}
+            - --config=/etc/adapter/config.yaml
+            - --v=2
+          ports:
+            - containerPort: 6443
+              name: https
+          resources:
+            {{- toYaml .Values.api.autoscaling.adapter.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/adapter
+            - name: tmp
+              mountPath: /tmp/cert
+      volumes:
+        - name: config
+          configMap:
+            name: {{ .Values.api.name }}-autoscaling-prometheus-adapter
+        - name: tmp
+          emptyDir: {}
+{{- end }}
diff --git a/braintrust/templates/api-autoscaling-prometheus.yaml b/braintrust/templates/api-autoscaling-prometheus.yaml
@@ -0,0 +1,123 @@
+{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }}
+{{- $scrapeApps := list .Values.api.name -}}
+{{- if .Values.api.workloadIsolation.enabled -}}
+{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.ingest.name -}}
+{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.background.name -}}
+{{- end -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.api.name }}-autoscaling-prometheus
+    namespace: {{ include "braintrust.namespace" . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }}
+      evaluation_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }}
+
+    scrape_configs:
+      - job_name: braintrust-api-autoscaling
+        kubernetes_sd_configs:
+          - role: pod
+            namespaces:
+              names:
+                - {{ include "braintrust.namespace" . }}
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_label_app]
+            regex: {{ join "|" $scrapeApps }}
+            action: keep
+          - source_labels: [__meta_kubernetes_pod_ip]
+            target_label: __address__
+            replacement: $1:{{ .Values.api.healthServer.port }}
+          - source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+        metrics_path: {{ .Values.api.autoscaling.metricsPath }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  selector:
+    app: {{ .Values.api.name }}-autoscaling-prometheus
+  ports:
+    - name: http
+      port: 9090
+      targetPort: 9090
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.api.name }}-autoscaling-prometheus
+  namespace: {{ include "braintrust.namespace" . }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Values.api.name }}-autoscaling-prometheus
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.api.name }}-autoscaling-prometheus
+    spec:
+      serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus
+      containers:
+        - name: prometheus
+          image: "{{ .Values.api.autoscaling.prometheus.image.repository }}:{{ .Values.api.autoscaling.prometheus.image.tag }}"
+          imagePullPolicy: {{ .Values.api.autoscaling.prometheus.image.pullPolicy }}
+          args:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention.time={{ .Values.api.autoscaling.prometheus.retention }}
+            - --web.enable-lifecycle
+          ports:
+            - containerPort: 9090
+              name: http
+          resources:
+            {{- toYaml .Values.api.autoscaling.prometheus.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/prometheus
+            - name: data
+              mountPath: /prometheus
+      volumes:
+        - name: config
+          configMap:
+            name: {{ .Values.api.name }}-autoscaling-prometheus
+        - name: data
+          emptyDir: {}
+{{- end }}
diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml
@@ -47,6 +47,7 @@ spec:
         target:
           type: Utilization
           averageUtilization: {{ $api.autoscaling.cpu.targetAverageUtilization }}
+    {{- if eq $root.Values.cloud "google" }}
     - type: Pods
       pods:
         metric:
@@ -61,6 +62,22 @@ spec:
         target:
           type: AverageValue
           averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }}
+    {{- else if eq $root.Values.cloud "aws" }}
+    - type: Pods
+      pods:
+        metric:
+          name: event_loop_utilization_ratio
+        target:
+          type: AverageValue
+          averageValue: {{ $api.autoscaling.eventLoopUtilization.targetAverageValue | quote }}
+    - type: Pods
+      pods:
+        metric:
+          name: event_loop_delay_mean_seconds
+        target:
+          type: AverageValue
+          averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }}
+    {{- end }}
   {{- with $api.autoscaling.behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}