diff --git a/braintrust/README.md b/braintrust/README.md index 435347c..91cd217 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -243,6 +243,12 @@ api: When enabled for a pool, that pool's `replicas` setting is ignored and the HPA controls the replica count. With `api.workloadIsolation.enabled`, ingest and background pools inherit these settings and can override `minReplicas` / `maxReplicas` under `api.workloadIsolation..autoscaling`. +## EKS API Autoscaling + +On AWS (`cloud: aws`), the same `api.autoscaling` values deploy an in-chart Prometheus scrape of each API pool's health `/metrics` endpoint and a prometheus-adapter that exposes event-loop gauges to HPA via `custom.metrics.k8s.io`. Targets match GKE / ECS defaults (CPU 50% on the `api` container, event-loop utilization `0.4`, delay mean `0.05s`). + +Requires API image **v2.9.0+**. With workload isolation enabled, Prometheus scrapes every pool (`api.name`, ingest, and background). + ## API workload isolation `api.workloadIsolation.enabled` creates fixed-capacity `braintrust-api-ingest` diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index c5b85ab..c3ff105 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -147,15 +147,18 @@ http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ {{- end -}} {{/* -Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD). +Validate API autoscaling prerequisites. +GKE requires AutoscalingMetric (autoscaling.gke.io/v1beta1). +EKS uses in-chart Prometheus + prometheus-adapter (no GKE CRD). */}} {{- define "braintrust.apiAutoscaling.validate" -}} -{{- if ne .Values.cloud "google" }} -{{- fail "api.autoscaling is currently only supported when cloud is google (GKE)" }} -{{- end }} +{{- if eq .Values.cloud "google" }} {{- if not (.Capabilities.APIVersions.Has "autoscaling.gke.io/v1beta1") }} {{- fail "api.autoscaling requires the AutoscalingMetric API (autoscaling.gke.io/v1beta1). Use GKE 1.35.1 or later, or verify with: kubectl api-resources | grep autoscalingmetric. For helm template without a cluster, pass --api-versions=autoscaling.gke.io/v1beta1." }} {{- end }} +{{- else if ne .Values.cloud "aws" }} +{{- fail "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" }} +{{- end }} {{- end -}} {{/* diff --git a/braintrust/templates/api-autoscaling-metric.yaml b/braintrust/templates/api-autoscaling-metric.yaml index f04245f..16f5694 100644 --- a/braintrust/templates/api-autoscaling-metric.yaml +++ b/braintrust/templates/api-autoscaling-metric.yaml @@ -1,4 +1,5 @@ {{- $root := . -}} +{{- if eq $root.Values.cloud "google" -}} {{- $pools := include "braintrust.apiPools" . | fromYamlArray -}} {{- $validated := false -}} {{- $rendered := 0 -}} @@ -51,3 +52,4 @@ spec: {{- $rendered = add1 $rendered -}} {{- end -}} {{- end }} +{{- end }} diff --git a/braintrust/templates/api-autoscaling-prometheus-adapter.yaml b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml new file mode 100644 index 0000000..ca2d137 --- /dev/null +++ b/braintrust/templates/api-autoscaling-prometheus-adapter.yaml @@ -0,0 +1,159 @@ +{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +rules: + - apiGroups: [""] + resources: ["namespaces", "pods", "services"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus-adapter-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +data: + config.yaml: | + rules: + - seriesQuery: 'braintrust_api_event_loop_delay_mean_seconds{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^braintrust_api_(.*)" + as: "$1" + metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: 'braintrust_api_event_loop_utilization_ratio{namespace!="",pod!=""}' + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: "^braintrust_api_(.*)" + as: "$1" + metricsQuery: avg(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +spec: + selector: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + ports: + - name: https + port: 443 + targetPort: 6443 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} + port: 443 + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + namespace: {{ include "braintrust.namespace" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + template: + metadata: + labels: + app: {{ .Values.api.name }}-autoscaling-prometheus-adapter + spec: + serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus-adapter + containers: + - name: prometheus-adapter + image: "{{ .Values.api.autoscaling.adapter.image.repository }}:{{ .Values.api.autoscaling.adapter.image.tag }}" + imagePullPolicy: {{ .Values.api.autoscaling.adapter.image.pullPolicy }} + args: + - --secure-port=6443 + - --cert-dir=/tmp/cert + - --prometheus-url=http://{{ .Values.api.name }}-autoscaling-prometheus.{{ include "braintrust.namespace" . }}.svc:9090 + - --metrics-relist-interval={{ .Values.api.autoscaling.prometheus.scrapeInterval }} + - --config=/etc/adapter/config.yaml + - --v=2 + ports: + - containerPort: 6443 + name: https + resources: + {{- toYaml .Values.api.autoscaling.adapter.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/adapter + - name: tmp + mountPath: /tmp/cert + volumes: + - name: config + configMap: + name: {{ .Values.api.name }}-autoscaling-prometheus-adapter + - name: tmp + emptyDir: {} +{{- end }} diff --git a/braintrust/templates/api-autoscaling-prometheus.yaml b/braintrust/templates/api-autoscaling-prometheus.yaml new file mode 100644 index 0000000..be19129 --- /dev/null +++ b/braintrust/templates/api-autoscaling-prometheus.yaml @@ -0,0 +1,123 @@ +{{- if and .Values.api.autoscaling.enabled (eq .Values.cloud "aws") }} +{{- $scrapeApps := list .Values.api.name -}} +{{- if .Values.api.workloadIsolation.enabled -}} +{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.ingest.name -}} +{{- $scrapeApps = append $scrapeApps .Values.api.workloadIsolation.background.name -}} +{{- end -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "braintrust.namespace" . }}-{{ .Values.api.name }}-autoscaling-prometheus +subjects: + - kind: ServiceAccount + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +data: + prometheus.yml: | + global: + scrape_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }} + evaluation_interval: {{ .Values.api.autoscaling.prometheus.scrapeInterval }} + + scrape_configs: + - job_name: braintrust-api-autoscaling + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - {{ include "braintrust.namespace" . }} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + regex: {{ join "|" $scrapeApps }} + action: keep + - source_labels: [__meta_kubernetes_pod_ip] + target_label: __address__ + replacement: $1:{{ .Values.api.healthServer.port }} + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + metrics_path: {{ .Values.api.autoscaling.metricsPath }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +spec: + selector: + app: {{ .Values.api.name }}-autoscaling-prometheus + ports: + - name: http + port: 9090 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.api.name }}-autoscaling-prometheus + namespace: {{ include "braintrust.namespace" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.api.name }}-autoscaling-prometheus + template: + metadata: + labels: + app: {{ .Values.api.name }}-autoscaling-prometheus + spec: + serviceAccountName: {{ .Values.api.name }}-autoscaling-prometheus + containers: + - name: prometheus + image: "{{ .Values.api.autoscaling.prometheus.image.repository }}:{{ .Values.api.autoscaling.prometheus.image.tag }}" + imagePullPolicy: {{ .Values.api.autoscaling.prometheus.image.pullPolicy }} + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time={{ .Values.api.autoscaling.prometheus.retention }} + - --web.enable-lifecycle + ports: + - containerPort: 9090 + name: http + resources: + {{- toYaml .Values.api.autoscaling.prometheus.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + volumes: + - name: config + configMap: + name: {{ .Values.api.name }}-autoscaling-prometheus + - name: data + emptyDir: {} +{{- end }} diff --git a/braintrust/templates/api-hpa.yaml b/braintrust/templates/api-hpa.yaml index 4b2d97e..10991f1 100644 --- a/braintrust/templates/api-hpa.yaml +++ b/braintrust/templates/api-hpa.yaml @@ -47,6 +47,7 @@ spec: target: type: Utilization averageUtilization: {{ $api.autoscaling.cpu.targetAverageUtilization }} + {{- if eq $root.Values.cloud "google" }} - type: Pods pods: metric: @@ -61,6 +62,22 @@ spec: target: type: AverageValue averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- else if eq $root.Values.cloud "aws" }} + - type: Pods + pods: + metric: + name: event_loop_utilization_ratio + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopUtilization.targetAverageValue | quote }} + - type: Pods + pods: + metric: + name: event_loop_delay_mean_seconds + target: + type: AverageValue + averageValue: {{ $api.autoscaling.eventLoopDelayMean.targetAverageValue | quote }} + {{- end }} {{- with $api.autoscaling.behavior }} behavior: {{- toYaml . | nindent 4 }} diff --git a/braintrust/tests/api-autoscaling_test.yaml b/braintrust/tests/api-autoscaling_test.yaml index cb22f4c..01dc574 100644 --- a/braintrust/tests/api-autoscaling_test.yaml +++ b/braintrust/tests/api-autoscaling_test.yaml @@ -2,6 +2,8 @@ suite: test API autoscaling templates templates: - api-autoscaling-metric.yaml - api-hpa.yaml + - api-autoscaling-prometheus.yaml + - api-autoscaling-prometheus-adapter.yaml - api-deployment.yaml - api-configmap.yaml capabilities: @@ -34,18 +36,18 @@ tests: - hasDocuments: count: 0 - - it: should fail when autoscaling is enabled on a non-google cloud + - it: should fail when autoscaling is enabled on an unsupported cloud template: api-hpa.yaml values: - __fixtures__/base-values.yaml - - __fixtures__/aws-values.yaml + - __fixtures__/azure-values.yaml set: api.autoscaling.enabled: true release: namespace: "braintrust" asserts: - failedTemplate: - errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE)" + errorMessage: "api.autoscaling is currently only supported when cloud is google (GKE) or aws (EKS)" - it: should render AutoscalingMetric for GKE with event-loop gauges template: api-autoscaling-metric.yaml @@ -435,3 +437,138 @@ tests: - hasDocuments: count: 0 filterAware: true + + - it: should not render GKE AutoscalingMetric on AWS + template: api-autoscaling-metric.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should not render Prometheus stack on GKE + template: api-autoscaling-prometheus.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: google + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 0 + + - it: should render HPA on AWS with prometheus-adapter metric names + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: spec.metrics[0].type + value: ContainerResource + - equal: + path: spec.metrics[0].containerResource.container + value: api + - equal: + path: spec.metrics[1].pods.metric.name + value: event_loop_utilization_ratio + - equal: + path: spec.metrics[1].pods.target.averageValue + value: "0.4" + - equal: + path: spec.metrics[2].pods.metric.name + value: event_loop_delay_mean_seconds + - equal: + path: spec.metrics[2].pods.target.averageValue + value: "0.05" + + - it: should scrape health server metrics on AWS Prometheus + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: ":8001" + - matchRegex: + path: data["prometheus.yml"] + pattern: "metrics_path: /metrics" + - matchRegex: + path: data["prometheus.yml"] + pattern: "regex: braintrust-api" + + - it: should scrape all API pools on AWS when workload isolation is enabled + template: api-autoscaling-prometheus.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["prometheus.yml"] + pattern: "braintrust-api-ingest" + - matchRegex: + path: data["prometheus.yml"] + pattern: "braintrust-api-background" + + - it: should render three HPAs on AWS when workload isolation is enabled + template: api-hpa.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + api.workloadIsolation.enabled: true + release: + namespace: "braintrust" + asserts: + - hasDocuments: + count: 3 + + - it: should render prometheus-adapter rules for event-loop gauges on AWS + template: api-autoscaling-prometheus-adapter.yaml + documentSelector: + path: kind + value: ConfigMap + values: + - __fixtures__/base-values.yaml + - __fixtures__/aws-values.yaml + set: + api.autoscaling.enabled: true + release: + namespace: "braintrust" + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_delay_mean_seconds" + - matchRegex: + path: data["config.yaml"] + pattern: "braintrust_api_event_loop_utilization_ratio" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index d91754f..3785374 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -133,19 +133,20 @@ api: hpa: {} autoscalingMetric: {} replicas: 4 - # Autoscale API pools on CPU and Node.js event-loop metrics via AutoscalingMetric + HPA. + # Autoscale API pools on CPU and Node.js event-loop metrics. # Applies to every rendered API pool (default, and ingest/background when # workloadIsolation is enabled). Pool overrides can set minReplicas/maxReplicas # (or disable) under api.workloadIsolation..autoscaling. - # The GKE feature for custom autoscaling metrics is in Preview (Pre-GA). It requires: + # cloud=google (GKE): AutoscalingMetric CRD + HPA. Preview (Pre-GA); requires: # - Braintrust API / data plane v2.9.0+ (Prometheus /metrics on the API health server) # - GKE 1.35.1-gke.1396000 or later # - the Performance HPA profile and Autoscaling API # - roles/autoscaling.metricsWriter on all node service accounts # - the Autoscaling API in the service perimeter when using VPC Service Controls # Helm install/upgrade fails if autoscaling.gke.io/v1beta1 is not on the cluster. - # When enabled for a pool, that pool's replicas setting is ignored. # See: https://docs.cloud.google.com/kubernetes-engine/docs/how-to/expose-custom-metrics-autoscaling + # cloud=aws (EKS): in-chart Prometheus + prometheus-adapter + HPA (custom.metrics). + # When enabled for a pool, that pool's replicas setting is ignored. autoscaling: enabled: false minReplicas: 4 @@ -166,6 +167,31 @@ api: stabilizationWindowSeconds: 300 scaleUp: stabilizationWindowSeconds: 60 + # Used when cloud=aws (Prometheus + prometheus-adapter). + prometheus: + image: + repository: prom/prometheus + tag: v3.8.0 + pullPolicy: IfNotPresent + retention: 1h + scrapeInterval: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + adapter: + image: + repository: registry.k8s.io/prometheus-adapter/prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 256Mi # Optional fixed-capacity workload isolation. This creates dedicated API # Deployments and Services for ingestion and background work while retaining # api.name as the default pool for all remaining traffic. Public ingress must