diff --git a/.agents/skills/update-against-main/SKILL.md b/.agents/skills/update-against-main/SKILL.md new file mode 100644 index 0000000000..ab86694708 --- /dev/null +++ b/.agents/skills/update-against-main/SKILL.md @@ -0,0 +1,35 @@ +--- +name: update-against-main +description: Merge agent-substrate/substrate main into the kagent-dev/substrate fork's main branch, resolve conflicts, validate the result, and safely update the fork. Use only when explicitly synchronizing the fork's main branch with upstream main. Do not use for updating, rebasing, or resolving conflicts in feature branches or pull requests. +--- + +# Update Against Main + +This skill applies only to synchronizing the fork's `main` branch. Do not invoke it for a feature branch or PR merely because that branch is behind or conflicts with `main`. + +1. Confirm the worktree, current branch, tracking branch, and remotes. Do not disturb unrelated changes. +2. Fetch `origin/main` and `upstream/main`, inspect their divergence, and create a dated backup branch from `origin/main`. +3. Rebuild `main` from `upstream/main` by replaying only intentional fork feature commits in dependency order. Drop merge commits and fork commits superseded by upstream. +4. Resolve conflicts in favor of current upstream APIs while preserving the remaining fork features. Inspect the resulting diff and linear history. +5. Keep Helm charts synchronized with their corresponding manifests. When either changes, inspect and update the other while preserving intentional Helm templating and conditionals, then run `make verify-helm-template` and `make verify-crd-chart` and compare any relevant resources not covered by those checks. +6. Run `make test` and `make verify`. +7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing: + - Use a dedicated cluster name and kubeconfig; record the temporary assets created by this run. Before recreating with `hack/create-kind-cluster.sh`, delete any old cluster owned by this sync using `hack/kind.sh delete cluster --name "$cluster_name"` with its dedicated `KUBECONFIG`. + - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway`. + - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane. + - Deploy the gVisor counter demo and both standard egress demos. + - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color` + - The full micro-VM suite with the CI environment: `E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh -v -args --no-color` + - Switch egress to agentgateway sdsmint, then run the MITM trust and targeted networking lanes for both runtimes exactly as the workflow specifies. + - Verify the live router and egress workloads use agentgateway. Never use Envoy for fork validation. +8. Treat `go test ./internal/e2e/...` without `-args --e2e` as compilation/package testing, not E2E coverage. +9. Do not push when unit, verification, or E2E checks fail or cannot run. Report the exact blocker instead. +10. After all checks pass, verify the worktree and rewritten commits, then update the fork with `git push --force-with-lease origin main`. Never use an unguarded force push. +11. Clean up temporary assets before finishing, including on failure or cancellation: + - Stop this run's test/install processes and port-forwards. Save any diagnostics needed to explain failures before tearing down workloads. + - Delete the task-owned Kind cluster with `hack/kind.sh delete cluster --name "$cluster_name"` using its dedicated `KUBECONFIG`. Verify both the cluster and its node containers are gone before removing the kubeconfig. + - Remove this run's disposable assets: generated micro-VM disks and images, downloaded bundles, build outputs, scratch scripts, and temporary kubeconfigs. Remove task-only Docker images, containers, and volumes once no longer in use. Preserve shared assets, caches, registries, and unrelated clusters; do not use global Docker prune commands. + - After a successful push, remove clean temporary worktrees with `git worktree remove` from another checkout. Preserve backup branches, unpushed commits, uncommitted changes, and diagnostics needed for unresolved failures. + - If teardown stalls (for example, Docker reports no exit event), inspect only the task's node containers, retry scoped deletion once, and report any remaining resources and exact blocker. Do not restart the global Docker daemon or kill unrelated processes. Report cleanup separately from validation so leftover assets are not hidden by passing tests. + +Use the current CI workflow as the source of truth for cluster setup, images, demos, runtime coverage, and environment variables, with the agentgateway-only override above. Never claim E2E passed unless workloads ran against the cluster. diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 0000000000..6a7438a89c --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,125 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + env: + VERSION: helm-e2e + E2E_ATENET_DATAPLANE: agentgateway + E2E_CREDENTIAL_PROVIDER: "1" + E2E_EGRESS_MITM: "1" + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Label nodes with the installed version + run: kubectl label nodes --all ate.dev/substrate-version=${VERSION} + - name: Create install namespace + run: kubectl create namespace ate-system + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + # Pushed under the image's real path (kagent-dev/substrate/): + # the chart composes {registry}/{repository}/{component}, so the local + # registry serves each image where the default repository expects it -- + # the same path-preserving rule a production mirror follows. + for component in ateapi atecontroller atelet podcertcontroller atenet credential-provider/kubernetes-secrets; do + KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component##*/}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + -f internal/e2e/suites/credentials/values.yaml \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Enable NFS + run: | + sudo modprobe nfs || true + sudo modprobe nfsd || true + - name: Install CSI NFS driver + run: hack/install-ate-kind.sh --setup-csi=nfs + - name: Deploy micro-VM counter demo + # The deploy creates the substrate ActorTemplate and waits for its golden + # snapshot internally; the ActorTemplate CRD (and its Ready condition) + # no longer exists to wait on. + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress-mitm + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_SANDBOX_CLASS: microvm + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/.github/workflows/helm-verify.yaml b/.github/workflows/helm-verify.yaml new file mode 100644 index 0000000000..7f4493997e --- /dev/null +++ b/.github/workflows/helm-verify.yaml @@ -0,0 +1,39 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Enforces the chart checks that were previously developer-run only: +# verify-helm-template's own comment says "Used in CI", but no workflow ran +# it -- so a chart change that broke hack/render-manifests.sh shipped a green +# PR and was caught in review by hand. +name: helm-verify +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + helm-verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-helm@v4 + - name: Lint the chart + run: helm lint charts/substrate charts/substrate-crds + - name: Verify committed manifests match the chart output + run: make verify-helm-template + - name: Install helm-unittest + run: helm plugin install https://github.com/helm-unittest/helm-unittest --version v1.0.2 + - name: Run the chart unit tests + run: helm unittest charts/substrate diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 965b3735bf..830a1647ef 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -60,15 +60,11 @@ jobs: e2e-test-matrix: name: E2E (${{ matrix.dataplane }}) runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability strategy: fail-fast: false matrix: include: - - dataplane: envoy - experimental: false - dataplane: agentgateway - experimental: true env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} steps: @@ -102,7 +98,7 @@ jobs: - name: Install Agent Substrate (${{ matrix.dataplane }}) # The dataplane selection applies to both the ingress router and egress # gateway. - run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} + run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | @@ -112,11 +108,8 @@ jobs: run: hack/install-ate-kind.sh --setup-csi=nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the - # counter-microvm demo onto the control plane installed above. The demo - # redeploys the control plane, so retain the selected dataplane. - env: - ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/run-microvm-demo-kind.sh + # counter-microvm demo onto the control plane installed above. + run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demos @@ -196,8 +189,7 @@ jobs: kubectl --context kind-kind get pods -A -l ate.dev/worker-pool \ -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ | while read -r ns name; do dump "$ns" "$name"; done - # Preserve the required-check name while the concrete Envoy and AgentGateway - # executions run as entries in the shared matrix above. + # Preserve the required-check name for the dataplane matrix above. e2e-test: name: e2e-test needs: e2e-test-matrix diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..5ccb325ad5 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,154 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet credential-provider/kubernetes-secrets; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component##*/}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + + - name: Build kubectl-ate release binaries + if: inputs.create_release + env: + VERSION: ${{ steps.tag.outputs.value }} + run: | + set -o errexit -o nounset -o pipefail + + mkdir -p dist + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ + -trimpath \ + -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -o "dist/kubectl-ate-${os}-${arch}" \ + ./cmd/kubectl-ate + done + done + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true + files: dist/kubectl-ate-* diff --git a/Makefile b/Makefile index ed26abe8ac..b04750a7dc 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ build: build-images build-atectl build-ate-setup .PHONY: build-images build-images: - $(KO) build $(KO_FLAGS) \ + $(KO) build --base-import-paths $(KO_FLAGS) \ --ldflags="$(LDFLAGS)" \ $(IMAGES) @@ -132,3 +132,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/charts/substrate-crds/Chart.yaml b/charts/substrate-crds/Chart.yaml new file mode 100644 index 0000000000..a69dcee0e9 --- /dev/null +++ b/charts/substrate-crds/Chart.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 0000000000..12fa31f0a7 --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml new file mode 100644 index 0000000000..ebc1473eae --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: csidriverconfigs.ate.dev +spec: + group: ate.dev + names: + kind: CSIDriverConfig + listKind: CSIDriverConfigList + plural: csidriverconfigs + shortNames: + - csidriverconfig + singular: csidriverconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.driverName + name: Driver + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CSIDriverConfig is the Schema for the csidriverconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + properties: + controllerEndpoint: + description: |- + ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). + pattern: ^(tcp|dns)://.+$ + type: string + driverName: + description: |- + DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + Matches the StorageClass referenced in ActorTemplate volume definitions. + maxLength: 63 + minLength: 1 + pattern: ^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ + type: string + nodeSocketOverride: + description: |- + NodeSocketOverride is an optional override for the CSI Node service socket + on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. + pattern: ^unix://.+$ + type: string + tls: + description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + properties: + enabled: + description: Enabled controls whether TLS is used. + type: boolean + serverName: + description: ServerName override for TLS verification. + type: string + usePodIdentity: + description: UsePodIdentity indicates whether to reuse Substrate's + Pod Identity (SPIFFE) certificates. + type: boolean + required: + - enabled + type: object + x-kubernetes-validations: + - message: tls.usePodIdentity must be true when tls.enabled is true; + manual certificates are not yet supported + rule: '!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)' + required: + - controllerEndpoint + - driverName + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 0000000000..22d46f5123 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,140 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced by an ActorTemplate's + sandbox_config.config_name (required) and decouples + sandbox binary selection from the workload definition. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must include a digest + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. An + ActorTemplate only uses SandboxConfigs whose SandboxClass matches its + sandbox_config.sandbox_class. + enum: + - gvisor + - microvm + type: string + required: + - pauseImage + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 0000000000..e1bc49a029 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,473 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement). The + concrete binary is still selected by WorkerImage. Defaults to gvisor. + The sandbox binaries themselves come from the SandboxConfig each + ActorTemplate names (required). + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + template: + description: Template holds optional metadata, scheduling, and resource + settings for worker workloads. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations are added to the generated Deployment and worker pods. Keys + in the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: annotation keys must be valid Kubernetes qualified + names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + labels: + additionalProperties: + description: |- + WorkerPoolLabelValue is a Kubernetes label value for generated worker + workloads. + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels are added to the generated Deployment and worker pods. Keys in + the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: label keys must be valid Kubernetes qualified names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + workerImage: + description: WorkerImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + required: + - replicas + - workerImage + type: object + status: + description: status is the observed state of WorkerPool + properties: + readyReplicas: + description: ReadyReplicas is the number of ready worker pods. + format: int32 + minimum: 0 + type: integer + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector is the label selector for the worker pods. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 0000000000..52bd748009 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 0000000000..7fd7b09ed1 --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,59 @@ +# substrate + +Helm chart for installing Agent Substrate. + +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# Install Substrate +helm upgrade --install substrate ./charts/substrate +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +The chart installs the Kubernetes credential provider and enables HTTPS egress +interception. Create the `egress-mitm-ca-pool` Secret and configure actor trust +as described in the [credential provider setup](../../docs/kubernetes-credential-provider.md). +Namespace grants default to an empty list, denying credential access. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | +| `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics, the actor lifecycle events and the router access log | +| `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | +| `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | +| `otel.traces.samplingRatio` | `0.01` | Fraction of parentless requests that start a trace, applied to the Go components and the router | +| `otel.metrics.enabled` | `true` | Sets the OTLP metrics exporter to `none`; the Go components do not honor this yet | +| `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | +| `otel.logs.enabled` | `true` | Set to `false` to export no logs. Gates both OTLP log sources: ateapi's actor lifecycle events and the router access log | +| `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 0000000000..c0e9875a45 --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,7 @@ +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL + +REQUIRED Kubernetes feature gates: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 0000000000..45184413fb --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,228 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name is "substrate" (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. + +The check is on the literal release name "substrate" rather than +$ctx.Chart.Name so this helper is context-safe: a parent chart can invoke it +with its own `.` (where .Chart.Name is the parent, not "substrate") and still +get the same prefixed name that this subchart's own templates render. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name "substrate" -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +ServiceAccount name of ate-api-server, as this chart creates it. Parent +charts that need to bind additional Roles to this SA (e.g. env-source +Secret/ConfigMap reads for ActorTemplate resolution) should reference this +helper instead of hardcoding "ate-api-server": + + {{ include "substrate.ateApiServer.serviceAccountName" . }} +*/}} +{{- define "substrate.ateApiServer.serviceAccountName" -}} +{{- include "substrate.fullname" (list "ate-api-server" .) -}} +{{- end -}} + +{{/* +gRPC endpoint that clients dial to reach ate-api-server. dns:/// scheme + +release-prefixed Service name + release namespace + :443. Suitable for +consumption as ATE_API_ENDPOINT / --ateapi-address: + + {{ include "substrate.ateApi.endpoint" . }} + -> dns:///-api..svc:443 +*/}} +{{- define "substrate.ateApi.endpoint" -}} +{{- printf "dns:///%s.%s.svc:443" (include "substrate.fullname" (list "api" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Plaintext HTTP URL that clients use to reach atenet-router. + + {{ include "substrate.atenetRouter.url" . }} + -> http://-atenet-router..svc:80 +*/}} +{{- define "substrate.atenetRouter.url" -}} +{{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +OTLP endpoint a signal exports to, or empty when the signal is disabled or no +endpoint resolves. The per-signal endpoint wins over the generic one, matching +the precedence the OpenTelemetry SDK gives OTEL_EXPORTER_OTLP__ENDPOINT +over OTEL_EXPORTER_OTLP_ENDPOINT. + +Usage: + {{ include "substrate.otel.signalEndpoint" (list "traces" .) }} +*/}} +{{- define "substrate.otel.signalEndpoint" -}} +{{- $signal := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $cfg := index $ctx.Values.otel $signal -}} +{{- if $cfg.enabled -}} +{{- $cfg.endpoint | default $ctx.Values.otel.endpoint -}} +{{- end -}} +{{- end -}} + +{{/* +OTEL_* env entries for a Go component, as a list of "- name/value" items. +Empty when nothing under .Values.otel is set, so callers can gate the env +key on the result. + +Usage: + {{- with include "substrate.otel.env" . }} + {{- . | trim | nindent 8 }} + {{- end }} +*/}} +{{- define "substrate.otel.env" -}} +{{- $otel := .Values.otel -}} +{{- if $otel.endpoint }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $otel.endpoint | quote }} +{{- end }} +{{- range $signal := list "traces" "metrics" "logs" }} +{{- $cfg := index $otel $signal }} +{{- if not $cfg.enabled }} +{{- /* "none" is the SDK's own exporter name for "export nothing"; leaving the + endpoint unset would fall back to the SDK default of localhost:4317. */}} +- name: OTEL_{{ upper $signal }}_EXPORTER + value: none +{{- else if $cfg.endpoint }} +- name: OTEL_EXPORTER_OTLP_{{ upper $signal }}_ENDPOINT + value: {{ $cfg.endpoint | quote }} +{{- end }} +{{- end }} +{{- if include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- /* Only logs need turning on: serverboot defaults the component to none, so + an enabled signal exports nothing without this. Traces and metrics always + export, so they need no such branch -- keep this out of the range above. + Gated on an endpoint, since otlp without one retries localhost:4317. */}} +- name: OTEL_LOGS_EXPORTER + value: otlp +{{- end }} +{{- if include "substrate.otel.signalEndpoint" (list "traces" .) }} +- name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio +- name: OTEL_TRACES_SAMPLER_ARG + value: {{ $otel.traces.samplingRatio | quote }} +{{- end }} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{/* image.registry used to carry the full prefix (ghcr.io/kagent-dev/substrate). + It is now the registry host only, joined onto image.repository -- the same + registry/repository split every kagent-family chart uses, so one + global.imageRegistry value redirects them all. A values file still carrying + a path in registry would render a doubled prefix that fails only at pod + start, so it fails the render here instead and names the split. */}} +{{- /* A scheme'd registry (ko://...) is hack/render-manifests.sh passing an + importpath prefix for `ko resolve` to substitute, same as the "" tag + sentinel below -- unambiguously not the old host+path shape, so the guard + lets it through. */ -}} +{{- if and (contains "/" $ctx.Values.image.registry) (not (contains "://" $ctx.Values.image.registry)) -}} +{{- fail (printf "image.registry (%q) carries a path. It is now the registry host only: keep the path in image.repository, e.g. registry: ghcr.io, repository: kagent-dev/substrate." $ctx.Values.image.registry) -}} +{{- end -}} +{{- $registry := printf "%s/%s" (default $ctx.Values.image.registry (($ctx.Values.global).imageRegistry)) $ctx.Values.image.repository -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} + +{{/* +Rewrite a full image reference ({registry}/{path}:{tag}) onto global.imageRegistry. + +The `images.*` values are single-string references, some digest-pinned, so the +mirror knob has to edit the string. The first path segment is a registry only when +it contains "." or ":" (the containerd rule); otherwise the reference is +docker.io-implied and the mirror is prefixed. The repository path is preserved +either way, so a mirror copies images under their existing paths. + +Usage: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }} +*/}} +{{- define "substrate.thirdPartyImage" -}} +{{- $ref := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $mirror := (($ctx.Values.global).imageRegistry) -}} +{{- if not $mirror -}} +{{- $ref -}} +{{- else -}} +{{- $parts := splitList "/" $ref -}} +{{- $first := first $parts -}} +{{- if and (gt (len $parts) 1) (or (contains "." $first) (contains ":" $first)) -}} +{{- printf "%s/%s" $mirror (join "/" (rest $parts)) -}} +{{- else -}} +{{- printf "%s/%s" $mirror $ref -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +imagePullSecrets for a pod spec: the chart's own list merged (union) with +global.imagePullSecrets. Renders nothing when both are empty. +*/}} +{{- define "substrate.imagePullSecrets" -}} +{{- $merged := concat (.Values.imagePullSecrets | default list) (((.Values.global).imagePullSecrets) | default list) | uniq -}} +{{- if $merged -}} +imagePullSecrets: +{{- toYaml $merged | nindent 0 }} +{{- end -}} +{{- end -}} + +{{/* +imagePullPolicy: global.imagePullPolicy when set, IfNotPresent otherwise. One +definition so the fallback cannot drift between pods. +*/}} +{{- define "substrate.imagePullPolicy" -}} +{{- ((.Values.global).imagePullPolicy) | default "IfNotPresent" -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..ca76ae3ef8 --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} +{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 0000000000..b93dbf4207 --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,228 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + terminationGracePeriodSeconds: 40 + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--postgres-connection-string=@env" + - "--postgres-schema=@env" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" + - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + - "--drain-delay=13s" + - "--drain-timeout=15s" +{{- with .Values.ateApi.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: actor-id-ca-pool + projected: + sources: + - secret: + name: actor-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 0000000000..dfd2fdab68 --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 0000000000..9e8588ceb2 --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,113 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" +{{- with include "substrate.otel.env" . }} + env: +{{- . | trim | nindent 8 }} +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 0000000000..5e89b2f2c1 --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,232 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] +# ClusterTrustBundles referenced by SystemInfo trustBundle data sources are +# resolved on the node: atelet reads them through an informer and projects +# the sanitized PEM into actors (see cmd/atelet/trustbundle.go). +- apiGroups: ["certificates.k8s.io"] + resources: ["clustertrustbundles"] + verbs: ["get", "watch", "list"] +# SandboxConfigs are watched to pre-download sandbox assets into the node's +# cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go). +- apiGroups: ["ate.dev"] + resources: ["sandboxconfigs"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ include "substrate.fullname" (list "ate-node-critical" .) }} +value: 1000000000 +globalDefault: false +description: "Node-local ate components that every actor activation depends on." +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + priorityClassName: {{ include "substrate.fullname" (list "ate-node-critical" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + resources: + requests: + cpu: 50m + memory: 128Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: device-plugins + mountPath: /var/lib/kubelet/device-plugins + - name: host-dev + mountPath: /host/dev + readOnly: true + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: device-plugins + hostPath: + path: /var/lib/kubelet/device-plugins + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml new file mode 100644 index 0000000000..b5cf552264 --- /dev/null +++ b/charts/substrate/templates/atenet-egress.yaml @@ -0,0 +1,287 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + frontendPolicies: + accessLog: + add: + substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the + # control plane is unreachable. + substrateEgressActorResolution: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + + binds: + - port: 8443 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/actor-id-ca-certs/ca.crt + routes: [] + - mode: internal + protocol: AUTO + listeners: + - protocol: HTTPS + tls: + mode: dynamicCa + cert: /run/egress-mitm/tls.crt + key: /run/egress-mitm/tls.key + routes: + - backends: + - dynamic: {} + policies: + backendTLS: {} + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + - protocol: HTTP + routes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + - protocol: TCP + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-egress +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-egress + template: + metadata: + labels: + app: atenet-egress + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-egress" .) }} + securityContext: + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + terminationGracePeriodSeconds: 60 + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: agentgateway + image: {{ include "substrate.thirdPartyImage" (list .Values.images.agentgateway .) }} + args: + - -f + - /etc/agentgateway/config.yaml + env: + - name: RUST_LOG + value: debug + ports: + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: stats + containerPort: 15020 + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 1 + volumeMounts: + - name: egress-mitm + mountPath: /run/egress-mitm + readOnly: true + - name: config + mountPath: /etc/agentgateway + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: ext-proc + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - router + - --mode=egress + - --namespace={{ .Release.Namespace }} + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + - --envoy-admin-address=localhost:15000 + - --atenet-dataplane=agentgateway + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + - name: metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + volumes: + - name: egress-mitm + secret: + secretName: egress-mitm-ca-pool + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - name: config + configMap: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + - name: drain-signal + emptyDir: {} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 0000000000..8d14854925 --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,382 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + statsAddr: 0.0.0.0:15020 + # Actor sandboxes behind a worker IP are replaced between requests. Do + # not retain an idle connection that may belong to the previous actor. + backend: + poolMaxSize: 0 + +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} + frontendPolicies: +{{- if $traces }} + tracing: + host: $AGENTGATEWAY_OTLP_TRACES_ADDRESS + protocol: grpc + randomSampling: {{ .Values.otel.traces.samplingRatio }} +{{- end }} +{{- if $logs }} + # Unsampled per-request access log. It carries the ate.* actor + # attribution, which the sampled traces cannot be aggregated on. + # `fields` stays unset so the OTLP sink inherits the full field set. + accessLog: + otlp: + host: $AGENTGATEWAY_OTLP_LOGS_ADDRESS + protocol: grpc +{{- end }} +{{- end }} + + backends: + - name: dynamic + dynamic: {} + policies: + backendTunnel: + proxy: + backend: /dynamic + mode: connect + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true + + gateways: + http: + port: 8080 + protocol: HTTP + https: + port: 8443 + protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + + routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors + gateways: + - http + - https + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "15020" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--mode=ingress" + - "--atenet-dataplane=agentgateway" + - "--namespace={{ .Release.Namespace }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--status-port=4040" + - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - name: agentgateway + image: {{ include "substrate.thirdPartyImage" (list .Values.images.agentgateway .) }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} + # agentgateway takes host:port, not a URL, so the scheme is trimmed. + env: +{{- if $traces }} + - name: AGENTGATEWAY_OTLP_TRACES_ADDRESS + value: {{ trimPrefix "http://" $traces | quote }} +{{- end }} +{{- if $logs }} + - name: AGENTGATEWAY_OTLP_LOGS_ADDRESS + value: {{ trimPrefix "http://" $logs | quote }} +{{- end }} +{{- end }} + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns-ca + readOnly: true + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP + - name: status + port: 4040 + targetPort: status + protocol: TCP + - name: stats + port: 15020 + targetPort: gw-metrics + protocol: TCP diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml new file mode 100644 index 0000000000..15012324c3 --- /dev/null +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -0,0 +1,167 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# The credential provider: a gRPC service that resolves ate-secret:// URIs +# of the k8s.io class to Kubernetes Secret values. It is the ONLY +# component in the egress credential-injection path with Kubernetes access; the +# egress gateway and the injector never read Secrets. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +--- +# The provider checks the actor's atespace-to-namespace grant before reading. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + template: + metadata: + annotations: + checksum/namespace-policy: {{ toJson .Values.credentialProvider.namespacePolicies | sha256sum }} + labels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + spec: + serviceAccountName: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + containers: + - name: k8s-credential-provider + image: {{ include "substrate.componentImage" (list "kubernetes-secrets" .) }} + imagePullPolicy: {{ include "substrate.imagePullPolicy" . }} + args: + - "--listen-address=:50051" + - "--metrics-address=:9090" + # Use this Service's DNS certificate; only the egress injector may call. + - "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + # Enforce the atespace→namespace authorization policy (default-deny). + - "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml" + - "--injector-spiffe-id=spiffe://cluster.local/ns/{{ .Release.Namespace }}/sa/{{ include "substrate.fullname" (list "atenet-egress" .) }}" + - "--log-level=info" + ports: + - name: grpc + containerPort: 50051 + - name: metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: metrics + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: namespace-policy + mountPath: /etc/k8s-credential-provider + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + volumes: + - name: namespace-policy + configMap: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-namespace-policy" .) }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + ports: + - name: grpc + port: 50051 + targetPort: grpc + protocol: TCP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-namespace-policy" .) }} + namespace: {{ .Release.Namespace }} +data: + namespace-policy.yaml: | + policies: {{ toJson .Values.credentialProvider.namespacePolicies }} diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 0000000000..073291828b --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,22 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 0000000000..b3fc65ee21 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,199 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 diff --git a/charts/substrate/templates/podmonitor.yaml b/charts/substrate/templates/podmonitor.yaml new file mode 100644 index 0000000000..af2a7956cd --- /dev/null +++ b/charts/substrate/templates/podmonitor.yaml @@ -0,0 +1,98 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +One PodMonitor per workload that serves metrics. Six do, from three sources, +and no two agree on the port name: + + ate-api-server, atelet, the atenet router, the atenet egress and the + Kubernetes credential provider start the metrics server themselves + (serverboot's StartMetricsServer). The first two name that port `prometheus`, + the other three name it `metrics`. + + ate-controller serves controller-runtime's own metrics server. Its + ctrl.Options sets no Metrics field, and only BindAddress "0" disables that + server, so it binds the default :8080 — the port its pod already declares as + `metrics`. This endpoint carries the controller_runtime_* and workqueue_* + families that docs/metrics/substrate.yaml lists under + bridged_metric_families. + + The router and the egress each run an agentgateway container of their own, + serving its stats on a second port: `gw-metrics` on the router, `stats` on + the egress. Those are the gateway's own metrics rather than the ate + registry's, and they are as uncollected as the rest, so each of those two + monitors carries both endpoints. + +Only the first three carry prometheus.io/* annotations, and an annotation is +read only by a Prometheus configured for pod-annotation discovery. A Prometheus +Operator installation discovers PodMonitors instead and ignores annotations, so +without these objects every one of these endpoints is served and none is +collected. + +A PodMonitor addresses a NAMED container port, so the name is per workload: one +shared name would match some pods and silently miss the others. The pod label +is per workload too — each carries a bare `app: ` except the +credential provider, whose label is the substrate.fullname helper's output and +so depends on the release name. + +podcertificate-controller gets no monitor: it emits no metrics, which +docs/metrics/substrate.yaml states under blind_spots. + +One object per workload rather than one selecting several: an interval and a +relabeling are a component's own, and the atelet is a DaemonSet whose series +are per node. +*/}} +{{- if .Values.metrics.podMonitor.enabled }} +{{- $components := list + (dict "app" "ate-api-server" "ports" (list "prometheus")) + (dict "app" "ate-controller" "ports" (list "metrics")) + (dict "app" "atelet" "ports" (list "prometheus")) + (dict "app" "atenet-egress" "ports" (list "metrics" "stats")) + (dict "app" "atenet-router" "ports" (list "metrics" "gw-metrics")) + (dict "app" "k8s-credential-provider" "ports" (list "metrics") "fullnameLabel" true) +}} +{{- range $components }} +{{- $label := .app }} +{{- if .fullnameLabel }} +{{- $label = include "substrate.fullname" (list .app $) }} +{{- end }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "substrate.fullname" (list .app $) }} + namespace: {{ $.Release.Namespace }} + labels: + app: {{ $label }} + {{- with $.Values.metrics.podMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: {{ $label }} + namespaceSelector: + matchNames: + - {{ $.Release.Namespace }} + podMetricsEndpoints: + {{- range .ports }} + - port: {{ . }} + path: /metrics + {{- with $.Values.metrics.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 0000000000..77df191879 --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,237 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.postgres.enabled }} +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + # Local socket access is limited to processes in this pod and is used by + # health checks, the workload's idempotent database bootstrap, and the + # tls-reloader sidecar's configuration reloads. + local all all trust + # PostgreSQL verifies client certificates against the pod-identity CA. It + # does not need its own serving CA because it never verifies its server certificate. + hostssl all all all trust clientcert=verify-ca + reload-tls.sh: | + # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup + # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod + # certificate in place about 30 minutes before it expires, so without this + # loop the server keeps presenting the certificate it booted with until it + # expires about a day later and every client stops trusting it. + set -eu + + # As PID 1 this shell only sees SIGTERM if a handler is installed, and only + # acts on it between commands, so the sleep below runs in the background + # and is waited on. Without both halves the pod takes the full termination + # grace period to go away. + trap 'exit 0' TERM INT + + CERT=/run/servicedns.podcert.ate.dev/credential-bundle.pem + CA=/run/podidentity.podcert.ate.dev/trust-bundle.pem + + # Comfortably inside the 30m headroom (notAfter - beginRefreshAt) that + # cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go + # leaves; hashing two small files costs nothing. + INTERVAL=60 + + reloaded="" + while true; do + current="$(sha256sum "${CERT}" "${CA}")" + # Reloading fails until the server is accepting connections, which is + # where every pod starts out, so only record a hash once it has worked. + # Starting empty also means a restart of this container costs one + # redundant reload rather than a missed one. + if [ "${current}" != "${reloaded}" ] \ + && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then + reloaded="${current}" + echo "$(date -u +%FT%TZ) reloaded TLS configuration" + fi + sleep "${INTERVAL}" & + wait $! + done +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + securityContext: + # Group ownership of the projected certificate below, and of the data + # volume so that a freshly provisioned one is writable. OnRootMismatch + # keeps the kubelet from walking the data directory on every start, + # which would leave PGDATA group-writable and postgres refusing to run. + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + # PostgreSQL re-reads its TLS files only on SIGHUP, so this sidecar + # reloads the server whenever the kubelet rotates the projected pod + # certificate. fsGroup is also what makes that projection readable: the + # kubelet writes it root-owned for as long as the pod's containers do not + # all agree on one non-root user, and grants the fsGroup group access, + # landing the key at root:postgres 0640, the only shared mode PostgreSQL + # accepts. Pinning runAsUser on the postgres container would make the key + # postgres-owned and group-readable, which it rejects. + # See https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-SETUP + initContainers: + - name: tls-reloader + restartPolicy: Always + image: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - /etc/postgresql/reload-tls.sh + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + resources: + requests: + cpu: 10m + memory: 32Mi + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: postgres + image: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + # 0600 plus the group read that fsGroup adds is the 0640 above. + defaultMode: 0600 + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The unix socket directory, shared so the sidecar can ask the running + # server to reload. The image defaults both the server and its clients to + # this path, so nothing else has to know about it. + - name: socket + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} +{{- end }} diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml new file mode 100644 index 0000000000..5a240f5baf --- /dev/null +++ b/charts/substrate/templates/role.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 0000000000..14854822af --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,139 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: rustfs + image: {{ include "substrate.thirdPartyImage" (list .Values.images.rustfs .) }} + imagePullPolicy: {{ include "substrate.imagePullPolicy" . }} + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + containers: + - name: create-bucket + image: {{ include "substrate.thirdPartyImage" (list .Values.images.awsCli .) }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 0000000000..286584eb01 --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,35 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with +# the platform so gVisor ActorTemplates have a config to name via +# sandboxConfig.configName. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" + arm64: + gvisor: + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 0000000000..baa8768453 --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,57 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/tests/podmonitor_test.yaml b/charts/substrate/tests/podmonitor_test.yaml new file mode 100644 index 0000000000..f166c124c8 --- /dev/null +++ b/charts/substrate/tests/podmonitor_test.yaml @@ -0,0 +1,183 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The PodMonitors of the six workloads that serve metrics. Two things here are +# easy to get wrong and silent when wrong, so each is pinned per workload: +# +# the port NAME, because a PodMonitor addresses a named container port and no +# two sources agree — ate-api-server and atelet name theirs `prometheus`, the +# others `metrics`, and the agentgateway sidecars of the router and the +# egress add `gw-metrics` and `stats`; +# +# the pod LABEL, because every workload carries a bare `app: ` +# except the credential provider, whose label is the substrate.fullname +# helper's output and so depends on the release name. +# +# A wrong name or label yields a monitor that matches nothing, and no error. +suite: metrics PodMonitors +templates: +- podmonitor.yaml +tests: +- it: renders nothing by default + asserts: + - hasDocuments: + count: 0 + +- it: renders one PodMonitor per workload that serves metrics + set: + metrics.podMonitor.enabled: true + asserts: + - hasDocuments: + count: 6 + +- it: addresses the port name ate-api-server's pod declares + set: + metrics.podMonitor.enabled: true + documentIndex: 0 + asserts: + - isKind: + of: PodMonitor + - equal: + path: spec.selector.matchLabels.app + value: ate-api-server + - equal: + path: spec.podMetricsEndpoints[0].port + value: prometheus + - equal: + path: spec.podMetricsEndpoints[0].path + value: /metrics + +- it: addresses ate-controller's controller-runtime metrics port + set: + metrics.podMonitor.enabled: true + documentIndex: 1 + asserts: + - equal: + path: spec.selector.matchLabels.app + value: ate-controller + - equal: + path: spec.podMetricsEndpoints[0].port + value: metrics + +- it: addresses the port name the atelet's pod declares + set: + metrics.podMonitor.enabled: true + documentIndex: 2 + asserts: + - equal: + path: spec.selector.matchLabels.app + value: atelet + - equal: + path: spec.podMetricsEndpoints[0].port + value: prometheus + +- it: scrapes both the egress ext-proc and its agentgateway sidecar + set: + metrics.podMonitor.enabled: true + documentIndex: 3 + asserts: + - equal: + path: spec.selector.matchLabels.app + value: atenet-egress + - equal: + path: spec.podMetricsEndpoints[0].port + value: metrics + - equal: + path: spec.podMetricsEndpoints[1].port + value: stats + +- it: scrapes both the router and its agentgateway sidecar + set: + metrics.podMonitor.enabled: true + documentIndex: 4 + asserts: + - equal: + path: spec.selector.matchLabels.app + value: atenet-router + - equal: + path: spec.podMetricsEndpoints[0].port + value: metrics + - equal: + path: spec.podMetricsEndpoints[1].port + value: gw-metrics + +# The credential provider's pod label is the substrate.fullname helper's +# output, not the bare component name: bare under the canonical release name +# `substrate`, release-prefixed under any other. Both are pinned, because a +# label that does not track the helper yields a monitor matching nothing. +- it: selects the credential provider by the bare name under the canonical release + set: + metrics.podMonitor.enabled: true + release: + name: substrate + documentIndex: 5 + asserts: + - equal: + path: spec.selector.matchLabels.app + value: k8s-credential-provider + - equal: + path: spec.podMetricsEndpoints[0].port + value: metrics + +- it: selects the credential provider by the prefixed name under any other release + set: + metrics.podMonitor.enabled: true + release: + name: ate + documentIndex: 5 + asserts: + - equal: + path: metadata.name + value: ate-k8s-credential-provider + - equal: + path: spec.selector.matchLabels.app + value: ate-k8s-credential-provider + +- it: scopes every monitor to the release namespace + set: + metrics.podMonitor.enabled: true + release: + namespace: ate-system + asserts: + - equal: + path: metadata.namespace + value: ate-system + - equal: + path: spec.namespaceSelector.matchNames[0] + value: ate-system + +- it: leaves the interval out when it is unset, so Prometheus' own default holds + set: + metrics.podMonitor.enabled: true + asserts: + - notExists: + path: spec.podMetricsEndpoints[0].interval + +- it: carries the interval on every endpoint, and the discovery labels + set: + metrics.podMonitor.enabled: true + metrics.podMonitor.interval: 60s + metrics.podMonitor.labels: + release: prometheus + documentIndex: 4 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].interval + value: 60s + - equal: + path: spec.podMetricsEndpoints[1].interval + value: 60s + - equal: + path: metadata.labels.release + value: prometheus diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 0000000000..1ebb9bed12 --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,151 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +postgres: + enabled: true + storageSize: 1Gi + connectionString: "" + schema: public + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + +# Additional arguments appended to the ateapi defaults. +ateApi: + extraArgs: [] + +# Kubernetes Secret provider and AGW HTTP/HTTPS credential injection. +# Includes get-only Secret RBAC. HTTPS requires egress-mitm-ca-pool and actor trust. +credentialProvider: + namespacePolicies: [] + # - atespace: team-a + # allowedNamespaces: [team-a-secrets] + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +# Prometheus metrics. Six workloads serve /metrics on a pod port: ate-api-server, +# atelet, the atenet router, the atenet egress and the credential provider start +# the metrics server themselves, and ate-controller serves controller-runtime's +# own on :8080, its default. The router and the egress each add a second port for +# their agentgateway container's stats. +# +# Only the first three carry prometheus.io/* annotations, which a Prometheus reads +# only when it does pod-annotation discovery. A Prometheus Operator installation +# discovers PodMonitors instead, so turn these on there or every endpoint is +# served and none is collected. +metrics: + podMonitor: + # Requires the monitoring.coreos.com/v1 CRDs. Off by default: the object + # does not apply on a cluster without the Prometheus Operator. + enabled: false + # Scrape interval. Empty leaves Prometheus' own default. + interval: "" + # Labels on every PodMonitor. Set whatever the Prometheus' or the agent's + # podMonitorSelector matches on, and any tenant label a multi-tenant + # backend routes by. + labels: {} + +# OTLP export, following the OpenTelemetry SDK environment variable spec. +# endpoint applies to every signal; a signal's own endpoint overrides it. +# A signal exports only when it is enabled and resolves to an endpoint. +# +# traces.enabled and metrics.enabled set OTEL__EXPORTER=none on the +# Go components, which do not read it yet and keep exporting; the setting +# takes effect on the router's agentgateway only. logs.enabled works in full, +# and gates both OTLP log sources: ateapi's actor lifecycle events and the +# router's agentgateway access log. +otel: + endpoint: "" + traces: + enabled: true + endpoint: "" + samplingRatio: 0.01 + metrics: + enabled: true + endpoint: "" + logs: + enabled: true + endpoint: "" + +# Values under `global` are visible to this chart and to every subchart. A parent +# chart or an operator sets one value here instead of one value per chart. +global: + # -- Mirror registry that overrides where every image is pulled from. This is + # the air-gap knob. It wins over image.registry for component images. Each + # `images.*` reference has its registry segment replaced. Repository paths are + # preserved, so a mirror only has to copy images under their existing paths. + # For control without the override, leave this unset and set image.registry. + imageRegistry: "" + # -- Pull secrets merged (union) into each pod's own imagePullSecrets list. + imagePullSecrets: [] + # -- Fallback imagePullPolicy where a container does not set one. + imagePullPolicy: "" + +# Pull secrets for every pod this chart renders. Merged with global.imagePullSecrets. +imagePullSecrets: [] + +image: + # Registry host for the component images, and nothing else. To change + # environments, change only this value or global.imageRegistry, which overrides + # it. A path inside `registry` fails the render, and the error names this + # split. + registry: ghcr.io + # Image path prefix under the registry, ahead of each component name. The path + # is identical on every registry that serves the images. A mirror copies the + # images under this same path. + repository: kagent-dev/substrate + tag: "" + +images: + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e + busybox: busybox:1.36 diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 88b2f43561..03df442400 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -25,6 +25,7 @@ import ( "github.com/agent-substrate/substrate/internal/atelet" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -32,6 +33,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -70,6 +72,11 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithInsecureCredentials disables transport security for local clusters without Pod Certificates. +func WithInsecureCredentials() DialerOption { + return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. func NewAteletDialer(ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { @@ -164,7 +171,7 @@ func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls if err != nil { return nil, fmt.Errorf("while loading CA bundle from %s: %w", serverCAPath, err) } - expectedID, err := spiffeid.FromSegments(trustDomain, "ns", ateletNamespace, "sa", ateletSA) + expectedID, err := spiffeid.FromSegments(trustDomain, "ns", installdefaults.NamespaceFromPodEnv(), "sa", ateletSA) if err != nil { return nil, fmt.Errorf("while building expected atelet SPIFFE ID: %w", err) } diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 6bc640146a..2fc2910cad 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" @@ -41,6 +42,22 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { + secure := NewAteletDialer(nil, "", "") + if _, err := secure.dialCredentials("pod-uid"); err == nil { + t.Fatal("secure dialer accepted empty credential paths") + } + + insecureDialer := NewAteletDialer(nil, "", "", WithInsecureCredentials()) + creds, err := insecureDialer.dialCredentials("pod-uid") + if err != nil { + t.Fatalf("insecure dial credentials: %v", err) + } + if got := creds.Info().SecurityProtocol; got != "insecure" { + t.Fatalf("security protocol = %q, want insecure", got) + } +} + // makeTestCA mints a self-signed CA and returns it along with an X.509 bundle // containing it as the sole authority for the cluster.local trust domain. func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { @@ -170,7 +187,7 @@ func TestDialForAteletOnNodeTarget(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{PodIPs: []corev1.PodIP{{IP: tc.ateletIP}}}, } @@ -191,7 +208,7 @@ func TestDialForAteletOnNodeTarget(t *testing.T) { func TestDialForAteletOnNodeNoIPs(t *testing.T) { ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-abc", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-abc", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } d := dialerWithAtelets(t, ateletPod) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index e78504b0c1..52b2f4996f 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore/objectstoretest" @@ -65,10 +66,8 @@ const ( // templates hand out. No object store is wired up behind it. testStorageLocation = "gs://fake-fake-fake" - // ateletNamespace and byNode mirror the unexported constants controlapi's - // atelet informer is built with. - ateletNamespace = "ate-system" - byNode = "by-node" + // byNode mirrors the unexported index name controlapi's atelet informer uses. + byNode = "by-node" ) var ( @@ -131,7 +130,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu } // 3. Initialize Informers - ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient) + ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient, installdefaults.SystemNamespace) scFactory := informers.NewSharedInformerFactory(k8sClient, 0) scLister := scFactory.Storage().V1().StorageClasses().Lister() @@ -219,6 +218,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu dialer, instruments, "", + 30*time.Second, volPlugins, objectStore, "https://nonexistent-issuer.example", @@ -708,7 +708,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ateletNamespace, + Namespace: installdefaults.SystemNamespace, Labels: map[string]string{"app": "atelet"}, }, Spec: corev1.PodSpec{ @@ -716,7 +716,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, }, } - created, err := kc.CoreV1().Pods(ateletNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + created, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -725,7 +725,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { } created.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} created.Status.Phase = corev1.PodRunning - if _, err := kc.CoreV1().Pods(ateletNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + if _, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("updating atelet pod %s status: %w", name, err) } return nil @@ -742,7 +742,7 @@ func setupAteletOnNode(t *testing.T, tc *testContext, name, nodeName string) { t.Fatalf("%v", err) } t.Cleanup(func() { - _ = tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ + _ = tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ GracePeriodSeconds: ptr.To[int64](0), }) }) diff --git a/cmd/ateapi/internal/controlapi/informer.go b/cmd/ateapi/internal/controlapi/informer.go index eeb7f3a501..a76d2300cc 100644 --- a/cmd/ateapi/internal/controlapi/informer.go +++ b/cmd/ateapi/internal/controlapi/informer.go @@ -23,12 +23,12 @@ import ( ) const ( - ateletNamespace = "ate-system" - byNode = "by-node" + byNode = "by-node" ) -// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for Atelet pods. -func AteletInformer(kc kubernetes.Interface) (informers.SharedInformerFactory, cache.SharedIndexInformer) { +// AteletInformer creates a SharedInformerFactory and SharedIndexInformer for +// Atelet pods in the given namespace. +func AteletInformer(kc kubernetes.Interface, ateletNamespace string) (informers.SharedInformerFactory, cache.SharedIndexInformer) { factory := informers.NewSharedInformerFactoryWithOptions(kc, 0, informers.WithNamespace(ateletNamespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index b433515586..c8633eac6a 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "sync" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -63,14 +64,8 @@ type VolumePluginRegistry interface { GetPlugin(ctx context.Context, name string) (volume.VolumePluginControlPlane, error) } -// NewRPCService creates an instance of the ControlServer service. This is what -// implements the outward-facing RPC interface. -// -// instruments may be nil; the record helpers no-op. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewRPCService creates an RPC service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. instruments and objectStore may be nil. func NewRPCService( persistence store.Interface, workerCache *workercache.Cache, @@ -80,6 +75,7 @@ func NewRPCService( dialer *AteletDialer, instruments *Instruments, egressGatewayAddress string, + actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, objectStore objectstore.Store, actorIdentityJWTIssuer string, @@ -101,7 +97,7 @@ func NewRPCService( actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, objectStore) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStore) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index dcc738e848..4a3b1c52e0 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -110,14 +111,13 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry + workflowDeadline time.Duration objectStore objectstore.Store } -// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end; instruments and objectStore +// may be nil. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -127,6 +127,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, + workflowDeadline time.Duration, objectStore objectstore.Store, ) *ActorWorkflow { return &ActorWorkflow{ @@ -139,6 +140,7 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, + workflowDeadline: workflowDeadline, objectStore: objectStore, } } @@ -213,7 +215,14 @@ func acquireLease(ctx context.Context, holder leaseHolder, key, subject string) } func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) { - return acquireLease(ctx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + workflowCtx, cancel := context.WithTimeout(ctx, w.workflowDeadline) + leaseCtx, lease, err := acquireLease(workflowCtx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + if err != nil { + cancel() + return nil, nil, err + } + context.AfterFunc(lease.Context(), cancel) + return leaseCtx, lease, nil } func acquireTagLease(ctx context.Context, holder leaseHolder, tagRef resources.TagRef) (context.Context, *store.Lease, error) { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 7240238d2e..dba7369bd0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -336,7 +337,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) objects := objectstoretest.New() - w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -432,7 +433,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { }) } - actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) // Suspend the actor as far as it gets: MarkSuspending mints the // in-progress URI, and the checkpoint writes under it actor, err := actorWorkflow.ensureMarkedSuspending(ctx, actorRef, actor, template) diff --git a/cmd/ateapi/internal/controlapi/workflow_lease_test.go b/cmd/ateapi/internal/controlapi/workflow_lease_test.go new file mode 100644 index 0000000000..b17f0583d1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_lease_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" +) + +type leaseStore struct{ store.Interface } + +func (leaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) { + return store.NewLease(ctx, func() {}), nil +} + +func TestAcquireActorLeaseWorkflowDeadline(t *testing.T) { + w := &ActorWorkflow{store: leaseStore{}, workflowDeadline: 20 * time.Millisecond} + + ctx, lease, err := w.acquireActorLease(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"}) + if err != nil { + t.Fatalf("acquireActorLease: %v", err) + } + t.Cleanup(lease.Close) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("context error = %v, want DeadlineExceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("workflow context did not reach its deadline") + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 7c7352e5b8..36aa1d4bf0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -231,13 +231,16 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso nodeName = worker.GetNodeName() // Drop just this actor's assignment; any other actors the worker // hosts keep theirs. - _, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) + released, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) if err != nil { if errors.Is(err, store.ErrVersionConflict) { return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") } return nil, err } + if w.workerCache != nil { + w.workerCache.Observe(released) + } } // 2. Clear the actor's assignment, now that the worker is freed diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index f778d4efb7..77050c02f6 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -1265,7 +1266,7 @@ func newWireCaptureWorkflow(t *testing.T, persistence store.Interface) (*ActorWo }) ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: ateletNamespace, Name: "atelet-1", UID: "atelet-uid"}, + ObjectMeta: metav1.ObjectMeta{Namespace: installdefaults.SystemNamespace, Name: "atelet-1", UID: "atelet-uid"}, Spec: corev1.PodSpec{NodeName: "node-1"}, } dialer := NewAteletDialer(newTestAteletIndexer(t, ateletPod), "", "") diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index af78631005..df46c2dd9a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -19,6 +19,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -57,7 +58,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, objectstoretest.New()) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoretest.New()) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -65,7 +66,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, objectStore: objects}, objects + return &ActorWorkflow{store: persistence, workflowDeadline: time.Minute, objectStore: objects}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/internal/workercache/workercache.go b/cmd/ateapi/internal/workercache/workercache.go index 6281586e63..d42be04736 100644 --- a/cmd/ateapi/internal/workercache/workercache.go +++ b/cmd/ateapi/internal/workercache/workercache.go @@ -120,6 +120,12 @@ func (c *Cache) Forget(name string) { delete(c.workers, name) } +// Observe applies a worker returned by a successful store write immediately, +// without waiting for the corresponding watch event. +func (c *Cache) Observe(worker *ateapipb.Worker) { + c.applyEvent(store.WorkerEvent{Type: store.WorkerEventUpdated, Worker: worker}) +} + func (c *Cache) sync(ctx context.Context) (*store.WorkerWatch, error) { watch, err := c.store.WatchWorkers(ctx) if err != nil { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index d7b38cb7f1..2dae66f29e 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -38,6 +38,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/authz" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/localca" "github.com/agent-substrate/substrate/internal/localjwtauthority" "github.com/agent-substrate/substrate/internal/objectstore" @@ -82,11 +83,13 @@ var ( actorIDCAPoolFile = pflag.String("actor-id-ca-pool", "", "The file that contains the CA pool for signing actor JWTs") podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") + ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") @@ -196,7 +199,12 @@ func main() { sandboxConfigLister := ateFactory.Api().V1alpha1().SandboxConfigs().Lister() csiDriverConfigLister := ateFactory.Api().V1alpha1().CSIDriverConfigs().Lister() - ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset) + // atelet shares ateapi's namespace in every supported deployment topology, + // so we read it from Kubernetes' downward API rather than expose a flag. + ateletNamespace := installdefaults.NamespaceFromPodEnv() + slog.InfoContext(ctx, "Resolved atelet namespace", slog.String("atelet-namespace", ateletNamespace)) + + ateletPodInformerFactory, ateletPodInformer := controlapi.AteletInformer(clientset, ateletNamespace) scInformerFactory := informers.NewSharedInformerFactory(clientset, 0) storageClassLister := scInformerFactory.Storage().V1().StorageClasses().Lister() @@ -228,7 +236,11 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) + var dialerOpts []controlapi.DialerOption + if *ateletInsecure { + dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) + } + ateletDialer := controlapi.NewAteletDialer(ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile) if err != nil { @@ -249,6 +261,7 @@ func main() { ateletDialer, instruments, *egressGatewayAddress, + *actorWorkflowDeadline, volPlugins, objectStore, actorIdentityJWTIssuer, @@ -365,8 +378,10 @@ func logFlagValues(ctx context.Context) { slog.String("actor-id-ca-pool", *actorIDCAPoolFile), slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index 218a18c2d8..a7ed1b1cf5 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -22,4 +22,4 @@ package controllers //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 6155e49aa0..c3e740ab74 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -93,6 +93,7 @@ var ( ateapiAddress = pflag.String("ateapi-address", "k8s:///api.ate-system.svc:443", "ateapi gRPC target used by the credential broker.") ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ateapi.") ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ateapi certificate.") + grpcInsecure = pflag.Bool("grpc-insecure", false, "Serve gRPC without transport security. Intended only for local clusters without Pod Certificates.") gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -316,73 +317,76 @@ func main() { // it would never list or watch. Start is idempotent per informer — this // launches the new one and leaves the already-running ones untouched. ateFactory.Start(stopCh) - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) - if err != nil { - serverboot.Fatal(ctx, "Failed to build server TLS config", err) - } - ateletCert, err := credbundle.Parse(*grpcServerCredBundle) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - if ateletIdentity == nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) - } - - ateomFacingTLS := tlsCfg.Clone() - ateomFacingTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) - if err := os.Remove(ateompath.AteomSupportSocket); err != nil && !errors.Is(err, os.ErrNotExist) { - serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) - } - ateomFacingLis, err := net.Listen("unix", ateompath.AteomSupportSocket) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen for credential broker", err) - } - defer ateomFacingLis.Close() - if err := os.Chmod(ateompath.AteomSupportSocket, 0o600); err != nil { - serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + serverOpts := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), } + if *grpcInsecure { + slog.WarnContext(ctx, "Serving atelet gRPC without transport security") + } else { + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) + } + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsCfg))) - ateomFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(ateomFacingTLS))) + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() - ateletpb.RegisterAteomSupportServer(ateomFacingSrv, &ateomSupportServer{ - controlClient: ateapipb.NewControlClient(ateapiConn), - workers: ateapipb.NewWorkerServiceClient(ateapiConn), - }) - go func() { - if err := ateomFacingSrv.Serve(ateomFacingLis); err != nil { - serverboot.Fatal(ctx, "Failed to serve credential broker", err) + ateletCert, err := credbundle.Parse(*grpcServerCredBundle) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) } - }() + ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + if ateletIdentity == nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) + } + ateomFacingTLS := tlsCfg.Clone() + ateomFacingTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) + if err := os.Remove(ateompath.AteomSupportSocket); err != nil && !errors.Is(err, os.ErrNotExist) { + serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) + } + ateomFacingLis, err := net.Listen("unix", ateompath.AteomSupportSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for credential broker", err) + } + defer ateomFacingLis.Close() + if err := os.Chmod(ateompath.AteomSupportSocket, 0o600); err != nil { + serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + } + ateomFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(ateomFacingTLS))) + ateletpb.RegisterAteomSupportServer(ateomFacingSrv, &ateomSupportServer{ + controlClient: ateapipb.NewControlClient(ateapiConn), + workers: ateapipb.NewWorkerServiceClient(ateapiConn), + }) + go func() { + if err := ateomFacingSrv.Serve(ateomFacingLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve credential broker", err) + } + }() + } - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 71030d34d3..3511d3cc27 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -358,29 +358,21 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { prewarmMaxJitter = 0 t.Cleanup(func() { ateompath.StaticFilesDir, prewarmMaxJitter = origDir, origJitter }) - host := imageVolumeTestRegistry(t) - pauseRef := host + "/pause:3.10" - pushPauseImage(t, pauseRef) - content := []byte("runsc binary bytes") sha := fmt.Sprintf("%x", sha256.Sum256(content)) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.PauseImage = "" ctx := t.Context() client := fake.NewSimpleClientset(cfg) factory := externalversions.NewSharedInformerFactory(client, 0) informer := factory.Api().V1alpha1().SandboxConfigs().Informer() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } herder := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} // Handler first, informer start second, mirroring main: atelet startup // must never wait on this informer's sync, and the initial List replays // the pre-existing config into the handler as an Add. - if err := startSandboxAssetPrewarm(ctx, informer, herder, store, false); err != nil { + if err := startSandboxAssetPrewarm(ctx, informer, herder, nil, false); err != nil { t.Fatalf("startSandboxAssetPrewarm: %v", err) } stopCh := make(chan struct{}) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 10f8bcf566..2bf8edb124 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -891,8 +891,8 @@ func stopContainers(ctx context.Context, rcmd containerRuntime, containers []*at _ = rcmd.cmdKill(ctx, ctr.GetName(), "SIGKILL") _ = rcmd.cmdWait(ctx, ctr.GetName()) } - _ = rcmd.cmdKill(ctx, ocispec.PauseContainer, "SIGKILL") - _ = rcmd.cmdWait(ctx, ocispec.PauseContainer) + // Keep the sandbox alive for application deletion. cleanupContainers + // force-deletes the pause container after deleting the applications. } func cleanupContainers(ctx context.Context, rcmd containerRuntime, containers []*ateompb.Container) error { diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index c46373cd70..f8c7aaf9ab 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -17,13 +17,51 @@ package main import ( + "context" + "os" + "path/filepath" "reflect" "testing" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ocispec" + "github.com/agent-substrate/substrate/internal/proto/ateompb" ) +func TestCleanupKeepsSandboxAliveUntilApplicationsDeleted(t *testing.T) { + path := filepath.Join(t.TempDir(), "runsc") + // Model runsc's control socket: deleting an application fails once the + // sandbox has stopped, even when deletion is forced. + script := `#!/bin/sh +set -eu +shift 5 +case "$1" in +kill) + if [ "$2" = _pause ]; then touch "$0.stopped"; fi + ;; +delete) + if [ "$3" = _pause ]; then + touch "$0.stopped" + elif [ -e "$0.stopped" ]; then + exit 128 + fi + ;; +esac +` + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + r := &runsc{path: path, actorUID: "test-actor"} + containers := []*ateompb.Container{{Name: "counter"}} + stopContainers(context.Background(), r, containers) + if err := cleanupContainers(context.Background(), r, containers); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path + ".stopped"); err != nil { + t.Fatalf("sandbox was not stopped: %v", err) + } +} + func TestKillArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", diff --git a/cmd/credential-provider/kubernetes-secrets/kubeprovider.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go index ae04b730ca..6fae284d06 100644 --- a/cmd/credential-provider/kubernetes-secrets/kubeprovider.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go @@ -27,6 +27,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" "google.golang.org/grpc/codes" @@ -62,7 +63,7 @@ type SecretRef struct { Key string } -// ParseURI parses a ate-secret:// URI of the kubernetes.io provider. It +// ParseURI parses an ate-secret:// URI of the k8s.io provider. It // rejects any other scheme or provider name. func ParseURI(raw string) (SecretRef, error) { u, err := url.Parse(raw) @@ -78,15 +79,15 @@ func ParseURI(raw string) (SecretRef, error) { // The grammar is scheme/host/path only; a query or fragment means the caller // assumed a syntax this provider does not honor, so reject it rather than // silently ignore it. - if u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { - return SecretRef{}, fmt.Errorf("credential URI %q: query and fragment components are not allowed", raw) + if u.User != nil || u.RawQuery != "" || u.ForceQuery || strings.Contains(raw, "#") { + return SecretRef{}, fmt.Errorf("credential URI %q: user info, query, and fragment components are not allowed", raw) } // Reject percent-encoding in the path of secret uri. if u.EscapedPath() != u.Path { return SecretRef{}, fmt.Errorf("credential URI %q: path must not contain percent-encoding", raw) } - segments := strings.Split(strings.Trim(u.Path, "/"), "/") + segments := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") for i, s := range segments { if s == "" { return SecretRef{}, fmt.Errorf("credential URI %q: empty path segment %d", raw, i) @@ -104,11 +105,11 @@ func ParseURI(raw string) (SecretRef, error) { if len(tail) != 3 { return SecretRef{}, fmt.Errorf("credential URI %q: want %s///, got %d trailing segments", raw, LocalLocator, len(tail)) } - return SecretRef{ - Namespace: tail[0], - Name: tail[1], - Key: tail[2], - }, nil + ref := SecretRef{Namespace: tail[0], Name: tail[1], Key: tail[2]} + if len(validation.IsDNS1123Label(ref.Namespace)) != 0 || len(validation.IsDNS1123Subdomain(ref.Name)) != 0 || len(validation.IsConfigMapKey(ref.Key)) != 0 { + return SecretRef{}, fmt.Errorf("credential URI contains an invalid namespace, secret name, or key") + } + return ref, nil } // Server implements credproviderpb.CredentialProviderServer over the Kubernetes @@ -118,12 +119,10 @@ type Server struct { client kubernetes.Interface // nsAuth restricts which namespaces an atespace may resolve secrets from. - // Nil disables authorization (dev only): every URI namespace is allowed. nsAuth *NamespaceAuthorizer } -// NewServer builds a Kubernetes-backed credential provider. nsAuth enforces the -// atespace→namespace policy; pass nil to disable authorization (dev only). +// NewServer builds a Kubernetes credential provider with a default-deny policy. func NewServer(client kubernetes.Interface, nsAuth *NamespaceAuthorizer) *Server { return &Server{client: client, nsAuth: nsAuth} } @@ -154,7 +153,7 @@ func (s *Server) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecre if k8serrors.IsForbidden(err) { return nil, status.Errorf(codes.PermissionDenied, "not permitted to read secret %s/%s", ref.Namespace, ref.Name) } - return nil, status.Errorf(codes.Unavailable, "reading secret %s/%s: %v", ref.Namespace, ref.Name, err) + return nil, status.Error(codes.Unavailable, "could not read secret from Kubernetes") } value, err := selectKey(secret.Data, ref.Key) @@ -168,9 +167,6 @@ func (s *Server) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecre // the attested actor SPIFFE ID and denies unless the URI's namespace is in that // atespace's allowed list. func (s *Server) authorize(ctx context.Context, actorSpiffeID, namespace string) error { - if s.nsAuth == nil { - return nil - } actor, err := resources.ActorRefFromSPIFFEID(actorSpiffeID) if err != nil { slog.WarnContext(ctx, "credential request denied: unusable actor identity", slog.Any("err", err)) diff --git a/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go index ad8c435627..30976ee4dd 100644 --- a/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go @@ -16,11 +16,19 @@ package main import ( "context" + "errors" + "os" + "path/filepath" + "strings" "testing" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -52,6 +60,14 @@ func TestParseURI(t *testing.T) { {name: "percent-encoded separator", uri: "ate-secret://k8s.io/default/ns1/example-api/tok%2Fen", wantErr: true}, {name: "percent-encoding of any kind", uri: "ate-secret://k8s.io/default/ns1/example-api/tok%2Den", wantErr: true}, {name: "space in path", uri: "ate-secret://k8s.io/default/ns1/example-api/tok en", wantErr: true}, + + {name: "user info", uri: "ate-secret://user@k8s.io/default/ns1/api/token", wantErr: true}, + {name: "empty query", uri: "ate-secret://k8s.io/default/ns1/api/token?", wantErr: true}, + {name: "empty fragment", uri: "ate-secret://k8s.io/default/ns1/api/token#", wantErr: true}, + {name: "trailing slash", uri: "ate-secret://k8s.io/default/ns1/api/token/", wantErr: true}, + {name: "empty namespace", uri: "ate-secret://k8s.io/default//api/token", wantErr: true}, + {name: "invalid namespace", uri: "ate-secret://k8s.io/default/NS/api/token", wantErr: true}, + {name: "path traversal", uri: "ate-secret://k8s.io/default/ns1/../token", wantErr: true}, {name: "unparseable", uri: "://://", wantErr: true}, } for _, tc := range tests { @@ -163,9 +179,13 @@ func TestFetchSecretAuthorization(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - srv := NewServer(fake.NewSimpleClientset(secret), authz) + client := fake.NewSimpleClientset(secret) + srv := NewServer(client, authz) resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: tc.actorSpiffeID}) if tc.wantCode != codes.OK { + if len(client.Actions()) != 0 { + t.Fatal("denied request reached Kubernetes") + } if status.Code(err) != tc.wantCode { t.Fatalf("code = %v, want %v (err=%v)", status.Code(err), tc.wantCode, err) } @@ -180,14 +200,14 @@ func TestFetchSecretAuthorization(t *testing.T) { }) } - // With no authorizer configured, enforcement is bypassed entirely. - t.Run("nil authorizer bypasses", func(t *testing.T) { + // A missing authorizer must fail closed. + t.Run("nil authorizer denies", func(t *testing.T) { srv := NewServer(fake.NewSimpleClientset(secret), nil) if _, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{ Uri: "ate-secret://k8s.io/default/ns1/example-api/token", - ActorSpiffeId: "not-a-spiffe-uri", - }); err != nil { - t.Fatalf("nil authorizer should not enforce, got %v", err) + ActorSpiffeId: teamAURI, + }); status.Code(err) != codes.PermissionDenied { + t.Fatalf("nil authorizer should deny, got %v", err) } }) } @@ -199,6 +219,14 @@ func TestFetchSecret(t *testing.T) { "token": []byte("s3cr3t"), }, } + multiKey := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "multi", Namespace: "ns1"}, + Data: map[string][]byte{ + "a": []byte("aa"), + "b": []byte("bb"), + }, + } + tests := []struct { name string uri string @@ -210,6 +238,16 @@ func TestFetchSecret(t *testing.T) { uri: "ate-secret://k8s.io/default/ns1/example-api/token", want: "s3cr3t", }, + { + name: "key required", + uri: "ate-secret://k8s.io/default/ns1/example-api", + wantCode: codes.InvalidArgument, + }, + { + name: "no key, multiple keys", + uri: "ate-secret://k8s.io/default/ns1/multi", + wantCode: codes.InvalidArgument, + }, { name: "missing key", uri: "ate-secret://k8s.io/default/ns1/example-api/nope", @@ -220,22 +258,17 @@ func TestFetchSecret(t *testing.T) { uri: "ate-secret://k8s.io/default/ns1/absent/token", wantCode: codes.NotFound, }, - { - name: "remote form rejected", - uri: "ate-secret://k8s.io/cluster/remote-east/ns1/example-api/token", - wantCode: codes.InvalidArgument, - }, { name: "bad uri", - uri: "ate-secret://vault.io/default/ns1/example-api", + uri: "ate-secret://vault.io/ns1/example-api", wantCode: codes.InvalidArgument, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset(secret) - srv := NewServer(client, nil) - resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri}) + client := fake.NewSimpleClientset(secret, multiKey) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/my-actor"}) if tc.wantCode != codes.OK { if status.Code(err) != tc.wantCode { t.Fatalf("FetchSecret(%q) code = %v, want %v (err=%v)", tc.uri, status.Code(err), tc.wantCode, err) @@ -251,3 +284,80 @@ func TestFetchSecret(t *testing.T) { }) } } + +func TestLoadNamespaceAuthorizer(t *testing.T) { + for _, tc := range []struct { + name, policy string + wantErr bool + }{ + {name: "valid", policy: "policies:\n- atespace: team-a\n allowedNamespaces: [ns1]\n"}, + {name: "empty", policy: "policies: []"}, + {name: "unknown field", policy: "polices: []", wantErr: true}, + {name: "duplicate field", policy: "policies: []\npolicies: []", wantErr: true}, + {name: "missing atespace", policy: "policies: [{allowedNamespaces: [ns1]}]", wantErr: true}, + {name: "invalid namespace", policy: "policies: [{atespace: team-a, allowedNamespaces: ['*']}]", wantErr: true}, + {name: "malformed", policy: "policies: [", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "policy.yaml") + if err := os.WriteFile(path, []byte(tc.policy), 0600); err != nil { + t.Fatal(err) + } + auth, err := LoadNamespaceAuthorizer(path) + if (err != nil) != tc.wantErr { + t.Fatalf("LoadNamespaceAuthorizer: %v", err) + } + if err == nil && auth.Allowed("team-a", "ns1") != (tc.name == "valid") { + t.Fatal("unexpected namespace grant") + } + }) + } + if _, err := LoadNamespaceAuthorizer(filepath.Join(t.TempDir(), "absent")); err == nil { + t.Fatal("missing policy accepted") + } +} + +func TestFetchSecretKubernetesErrors(t *testing.T) { + for _, tc := range []struct { + name string + err error + code codes.Code + }{ + {"forbidden", k8serrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, "api", errors.New("RBAC")), codes.PermissionDenied}, + {"unavailable", errors.New("upstream response body should stay private"), codes.Unavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, tc.err }) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + _, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", + }) + if status.Code(err) != tc.code { + t.Fatalf("FetchSecret: %v, want %v", err, tc.code) + } + if strings.Contains(err.Error(), "stay private") { + t.Fatal("Kubernetes response body exposed") + } + }) + } +} + +func TestFetchSecretObservesRotation(t *testing.T) { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("first")}} + client := fake.NewSimpleClientset(secret) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + req := &credproviderpb.FetchSecretRequest{Uri: "ate-secret://k8s.io/default/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a"} + first, err := srv.FetchSecret(t.Context(), req) + if err != nil || string(first.GetOpaqueBytes()) != "first" { + t.Fatalf("first fetch: %v, %v", first, err) + } + secret.Data["token"] = []byte("rotated") + if _, err := client.CoreV1().Secrets("ns1").Update(t.Context(), secret, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + next, err := srv.FetchSecret(t.Context(), req) + if err != nil || string(next.GetOpaqueBytes()) != "rotated" { + t.Fatalf("fetch after rotation: %v, %v", next, err) + } +} diff --git a/cmd/credential-provider/kubernetes-secrets/main.go b/cmd/credential-provider/kubernetes-secrets/main.go index a0c880da6a..b7139c9d4c 100644 --- a/cmd/credential-provider/kubernetes-secrets/main.go +++ b/cmd/credential-provider/kubernetes-secrets/main.go @@ -25,7 +25,9 @@ import ( "fmt" "log/slog" "net" + "net/url" "os/signal" + "strings" "syscall" "time" @@ -45,20 +47,17 @@ import ( const serviceName = "credprovider" -// injectorSPIFFEID is the identity of the only caller allowed to fetch secrets: -// the egress gateway's credential injector. -const injectorSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atenet-egress" - var ( - listenAddr = pflag.String("listen-address", ":50051", "gRPC listen address") - metricsAddr = pflag.String("metrics-address", ":9090", "Prometheus/health HTTP listen address") - serverBundle = pflag.String("server-cred-bundle", "", "credential bundle (PEM key+chain) presented for serving TLS (required)") - clientCAFile = pflag.String("client-ca-file", "", "CA bundle that caller (injector) client certificates must chain to (required)") - nsPolicyFile = pflag.String("namespace-policy-file", "", "path to the atespace→namespace authorization YAML (required)") - logLevel = pflag.String("log-level", "info", "one of debug, info, warn, error") - drainGrace = pflag.Duration("drain-grace", 5*time.Second, "how long to wait for in-flight RPCs on shutdown before a hard stop") - kubeAPIQPS = pflag.Float32("kube-api-qps", 50, "Sustained queries per second allowed against the Kubernetes API.") - kubeAPIBurst = pflag.Int("kube-api-burst", 100, "Burst queries allowed against the Kubernetes API.") + injectorSPIFFEID = pflag.String("injector-spiffe-id", "spiffe://cluster.local/ns/ate-system/sa/atenet-egress", "SPIFFE identity of the egress injector allowed to fetch credentials") + listenAddr = pflag.String("listen-address", ":50051", "gRPC listen address") + metricsAddr = pflag.String("metrics-address", ":9090", "Prometheus/health HTTP listen address") + serverBundle = pflag.String("server-cred-bundle", "", "credential bundle (PEM key+chain) presented for serving TLS (required)") + clientCAFile = pflag.String("client-ca-file", "", "CA bundle that caller (injector) client certificates must chain to (required)") + nsPolicyFile = pflag.String("namespace-policy-file", "", "path to the atespace→namespace authorization YAML (required)") + logLevel = pflag.String("log-level", "info", "one of debug, info, warn, error") + drainGrace = pflag.Duration("drain-grace", 5*time.Second, "how long to wait for in-flight RPCs on shutdown before a hard stop") + kubeAPIQPS = pflag.Float32("kube-api-qps", 50, "Sustained queries per second allowed against the Kubernetes API.") + kubeAPIBurst = pflag.Int("kube-api-burst", 100, "Burst queries allowed against the Kubernetes API.") ) func main() { @@ -178,6 +177,11 @@ func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, er return nil, fmt.Errorf("--client-ca-file is required") } + id, err := url.Parse(*injectorSPIFFEID) + if err != nil || id.Scheme != "spiffe" || id.Host == "" || id.Path == "" || id.User != nil || id.RawQuery != "" || id.ForceQuery || strings.Contains(*injectorSPIFFEID, "#") { + return nil, fmt.Errorf("--injector-spiffe-id must be a SPIFFE URI") + } + // Load the client CA pool once so a missing or empty projection fails the // pod promptly; GetConfigForClient below reloads it for every connection. loadPool := credbundle.PoolLoader(*clientCAFile) @@ -186,7 +190,7 @@ func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, er } serverCert := credbundle.Loader(*serverBundle) - verifySAN := verifyClientSAN(injectorSPIFFEID) + verifySAN := verifyClientSAN(*injectorSPIFFEID) // GetConfigForClient builds the config anew per connection: a certificate // signed by a newly published CA verifies without a restart. @@ -206,7 +210,7 @@ func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, er }, } slog.InfoContext(ctx, "verifying caller client certificates", - slog.String("ca", *clientCAFile), slog.String("required_san", injectorSPIFFEID)) + slog.String("ca", *clientCAFile), slog.String("required_san", *injectorSPIFFEID)) return credentials.NewTLS(cfg), nil } diff --git a/cmd/credential-provider/kubernetes-secrets/main_test.go b/cmd/credential-provider/kubernetes-secrets/main_test.go index 66f65c561b..bc41f2829f 100644 --- a/cmd/credential-provider/kubernetes-secrets/main_test.go +++ b/cmd/credential-provider/kubernetes-secrets/main_test.go @@ -31,7 +31,13 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "google.golang.org/grpc" "google.golang.org/grpc/credentials" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" ) func certWithURIs(t *testing.T, uris ...string) *x509.Certificate { @@ -48,7 +54,7 @@ func certWithURIs(t *testing.T, uris ...string) *x509.Certificate { } func TestVerifyClientSAN(t *testing.T) { - const injector = injectorSPIFFEID + injector := *injectorSPIFFEID tests := []struct { name string @@ -118,8 +124,8 @@ func TestBuildServerCredsReloadsClientCA(t *testing.T) { t.Fatalf("buildServerCreds() error = %v", err) } - fromCA1 := clientCA1.issue(t, certOpts{uris: []string{injectorSPIFFEID}}) - fromCA2 := clientCA2.issue(t, certOpts{uris: []string{injectorSPIFFEID}}) + fromCA1 := clientCA1.issue(t, certOpts{uris: []string{*injectorSPIFFEID}}) + fromCA2 := clientCA2.issue(t, certOpts{uris: []string{*injectorSPIFFEID}}) wrongSAN := clientCA1.issue(t, certOpts{uris: []string{"spiffe://cluster.local/ns/ate-system/sa/impostor"}}) // Before rotation only CA1 is trusted. @@ -294,3 +300,116 @@ func writeFileWithMtime(t *testing.T, path string, data []byte, mtime time.Time) t.Fatalf("chtimes %s: %v", path, err) } } + +func TestProviderMTLS(t *testing.T) { + ca, err := localca.GenerateCA("trusted", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + untrustedCA, err := localca.GenerateCA("untrusted", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + servingCert := issueCertificate(t, ca, "") + dir := t.TempDir() + oldBundle, oldCAFile, oldInjector := *serverBundle, *clientCAFile, *injectorSPIFFEID + t.Cleanup(func() { *serverBundle, *clientCAFile, *injectorSPIFFEID = oldBundle, oldCAFile, oldInjector }) + *serverBundle, *clientCAFile = filepath.Join(dir, "server.pem"), filepath.Join(dir, "ca.pem") + *injectorSPIFFEID = "spiffe://cluster.local/ns/custom/sa/release-atenet-egress" + key, err := x509.MarshalPKCS8PrivateKey(servingCert.PrivateKey) + if err != nil { + t.Fatal(err) + } + bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: servingCert.Certificate[0]})...) + if err := os.WriteFile(*serverBundle, bundle, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(*clientCAFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw}), 0600); err != nil { + t.Fatal(err) + } + creds, err := buildServerCreds(t.Context()) + if err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("credential")}, + }) + srv := grpc.NewServer(grpc.Creds(creds)) + credproviderpb.RegisterCredentialProviderServer(srv, NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}})) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + for _, tc := range []struct { + name string + certs []tls.Certificate + allowed bool + }{ + {"injector", []tls.Certificate{issueCertificate(t, ca, *injectorSPIFFEID)}, true}, + {"other workload", []tls.Certificate{issueCertificate(t, ca, "spiffe://cluster.local/ns/custom/sa/other")}, false}, + {"missing certificate", nil, false}, + {"untrusted injector", []tls.Certificate{issueCertificate(t, untrustedCA, *injectorSPIFFEID)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: roots, ServerName: "api.ate-system.svc", Certificates: tc.certs, MinVersion: tls.VersionTLS13, + }))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + before := len(client.Actions()) + resp, err := credproviderpb.NewCredentialProviderClient(conn).FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", + }) + if tc.allowed { + if err != nil || string(resp.GetOpaqueBytes()) != "credential" { + t.Fatalf("FetchSecret: %v, %v", resp, err) + } + } else { + if err == nil { + t.Fatal("unauthorized peer received credentials") + } + if len(client.Actions()) != before { + t.Fatal("unauthorized peer reached Kubernetes") + } + } + }) + } +} + +func issueCertificate(t *testing.T, ca *localca.CA, uri string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: serial, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + DNSNames: []string{"api.ate-system.svc", "localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + } + if uri != "" { + parsed, err := url.Parse(uri) + if err != nil { + t.Fatal(err) + } + template.URIs = []*url.URL{parsed} + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, &key.PublicKey, ca.SigningKey) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} diff --git a/cmd/credential-provider/kubernetes-secrets/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go new file mode 100644 index 0000000000..c787636048 --- /dev/null +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -0,0 +1,309 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "errors" + "io" + "os/exec" + "reflect" + "slices" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestProviderManifests(t *testing.T) { + for _, tc := range []struct { + name, tool, namespace, prefix string + image string + args []string + }{ + {name: "default", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", namespace: "custom", prefix: "test-", + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, + {name: "kustomize", tool: "kubectl", namespace: "ate-system", + args: []string{"kustomize", "../../../manifests/egress-credential-injection"}}, + {name: "CI images", tool: "helm", namespace: "ate-system", + image: "localhost:5001/kagent-dev/substrate/kubernetes-secrets:helm-e2e", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", "--set", "image.registry=localhost:5001", "--set", "image.tag=helm-e2e"}}, + {name: "global images", tool: "helm", namespace: "ate-system", + image: "mirror.example/custom/substrate/kubernetes-secrets:test", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", + "--set", "image.repository=custom/substrate", "--set", "image.tag=test", "--set", "global.imageRegistry=mirror.example", + "--set", "imagePullSecrets[0].name=local", "--set", "global.imagePullSecrets[0].name=global", "--set", "global.imagePullPolicy=Always"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := exec.LookPath(tc.tool); err != nil { + t.Skipf("%s is not installed", tc.tool) + } + data, err := exec.CommandContext(t.Context(), tc.tool, tc.args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, data) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var providerFound, portFound, policyFound, accountFound bool + var roleFound, bindingFound bool + for { + var doc struct { + Kind string + Metadata metav1.ObjectMeta + Spec struct { + Template corev1.PodTemplateSpec + Ports []corev1.ServicePort + } + Data map[string]string + Rules []rbacv1.PolicyRule + RoleRef rbacv1.RoleRef + Subjects []rbacv1.Subject + } + if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + switch doc.Kind { + case "ServiceAccount": + if doc.Metadata.Name == tc.prefix+"k8s-credential-provider" { + accountFound = true + } + case "Deployment": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider" { + continue + } + pod := doc.Spec.Template.Spec + if pod.ServiceAccountName != tc.prefix+"k8s-credential-provider" { + t.Fatalf("unexpected ServiceAccount %q", pod.ServiceAccountName) + } + for _, container := range pod.Containers { + if container.Name != "k8s-credential-provider" { + continue + } + providerFound = true + if tc.image != "" && container.Image != tc.image { + t.Errorf("image = %q, want %q", container.Image, tc.image) + } + if tc.name == "global images" { + if container.ImagePullPolicy != corev1.PullAlways { + t.Errorf("imagePullPolicy = %q, want Always", container.ImagePullPolicy) + } + for _, name := range []string{"local", "global"} { + if !slices.Contains(pod.ImagePullSecrets, corev1.LocalObjectReference{Name: name}) { + t.Errorf("missing imagePullSecret %q", name) + } + } + } + args := strings.Join(container.Args, " ") + for _, required := range []string{ + "--listen-address=:50051", "--metrics-address=:9090", + "--injector-spiffe-id=spiffe://cluster.local/ns/" + tc.namespace + "/sa/" + tc.prefix + "atenet-egress", + "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem", + "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem", + } { + if tc.tool == "kubectl" && strings.HasPrefix(required, "--injector-spiffe-id=") { + continue + } + if !strings.Contains(args, required) { + t.Errorf("provider missing %s", required) + } + } + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet.Port.StrVal != "metrics" { + t.Fatal("missing dedicated readiness probe") + } + } + case "Service": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider" { + continue + } + for _, port := range doc.Spec.Ports { + if port.Port == 50051 && port.TargetPort.StrVal == "grpc" { + portFound = true + } + } + case "ConfigMap": + if !strings.HasPrefix(doc.Metadata.Name, tc.prefix+"k8s-credential-provider-namespace-policy") { + continue + } + policyFound = true + var policy namespacePolicyFile + if err := yaml.UnmarshalStrict([]byte(doc.Data["namespace-policy.yaml"]), &policy); err != nil { + t.Fatal(err) + } + auth, err := newNamespaceAuthorizer(policy) + if err != nil { + t.Fatal(err) + } + if auth.Allowed("team-a", "ns1") != (tc.name == "custom release") { + t.Fatal("unexpected namespace policy") + } + case "ClusterRole": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider-secret-reader" { + continue + } + roleFound = true + want := []rbacv1.PolicyRule{{APIGroups: []string{""}, Resources: []string{"secrets"}, Verbs: []string{"get"}}} + if !reflect.DeepEqual(doc.Rules, want) { + t.Fatalf("provider rules = %#v, want get-only Secret access", doc.Rules) + } + case "ClusterRoleBinding": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider-secret-reader" { + continue + } + bindingFound = true + wantRef := rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: tc.prefix + "k8s-credential-provider-secret-reader"} + wantSubjects := []rbacv1.Subject{{Kind: "ServiceAccount", Name: tc.prefix + "k8s-credential-provider", Namespace: tc.namespace}} + if doc.RoleRef != wantRef || !reflect.DeepEqual(doc.Subjects, wantSubjects) { + t.Fatalf("unexpected provider binding: roleRef=%+v subjects=%+v", doc.RoleRef, doc.Subjects) + } + } + } + if !providerFound || !portFound || !policyFound || !accountFound { + t.Fatalf("provider=%v port=%v policy=%v account=%v", providerFound, portFound, policyFound, accountFound) + } + if !roleFound || !bindingFound { + t.Fatalf("role=%v binding=%v", roleFound, bindingFound) + } + }) + } +} + +func TestAgentgatewayCredentialConfiguration(t *testing.T) { + for _, tc := range []struct { + name, tool, host, roots string + args []string + }{ + {name: "default", tool: "helm", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", host: "test-k8s-credential-provider.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom"}}, + {name: "kustomize", tool: "kubectl", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", + args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../../manifests/ate-install/agentgateway-egress-mitm"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := exec.LookPath(tc.tool); err != nil { + t.Skipf("%s is not installed", tc.tool) + } + data, err := exec.CommandContext(t.Context(), tc.tool, tc.args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, data) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + providers := map[string]int{} + mitmMounts, mitmVolumes, passthroughListeners := 0, 0, 0 + for { + var doc struct { + Kind string + Data map[string]string + Spec struct{ Template corev1.PodTemplateSpec } + } + if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + if doc.Kind == "Deployment" { + for _, volume := range doc.Spec.Template.Spec.Volumes { + if volume.Secret != nil && volume.Secret.SecretName == "egress-mitm-ca-pool" { + mitmVolumes++ + } + } + for _, container := range doc.Spec.Template.Spec.Containers { + if container.Name != "agentgateway" { + continue + } + for _, mount := range container.VolumeMounts { + if mount.MountPath == "/run/egress-mitm" { + mitmMounts++ + } + } + } + } + if doc.Kind != "ConfigMap" { + continue + } + var config struct { + Binds []struct { + Listeners []struct { + Protocol string + TLS struct{ Mode, Cert, Key string } + Routes []struct { + Backends []struct { + Dynamic map[string]any + Policies struct{ BackendTLS map[string]any } + } + Policies struct { + SubstrateEgress struct { + CredentialProviders []struct { + URIAuthority string `json:"uriAuthority"` + Target struct { + Host string + Policies struct { + BackendTLS struct{ Cert, Key, Root string } + } + } + } + } + } + } + } + } + } + if err := yaml.Unmarshal([]byte(doc.Data["config.yaml"]), &config); err != nil { + t.Fatal(err) + } + for _, bind := range config.Binds { + for _, listener := range bind.Listeners { + if listener.Protocol == "TLS" { + passthroughListeners++ + } + for _, route := range listener.Routes { + for _, provider := range route.Policies.SubstrateEgress.CredentialProviders { + providers[listener.Protocol]++ + if listener.Protocol == "HTTPS" { + if listener.TLS.Mode != "dynamicCa" || listener.TLS.Cert != "/run/egress-mitm/tls.crt" || listener.TLS.Key != "/run/egress-mitm/tls.key" { + t.Fatal("incorrect MITM configuration") + } + if len(route.Backends) != 1 || route.Backends[0].Dynamic == nil || len(route.Backends[0].Dynamic) != 0 || route.Backends[0].Policies.BackendTLS == nil || len(route.Backends[0].Policies.BackendTLS) != 0 { + t.Fatal("HTTPS must use a dynamic destination with default public TLS trust") + } + } else if listener.Protocol != "HTTP" { + t.Fatalf("credentials enabled on unexpected protocol %q", listener.Protocol) + } + if provider.URIAuthority != "k8s.io" || provider.Target.Host != tc.host { + t.Fatalf("incorrect provider: %+v", provider) + } + tls := provider.Target.Policies.BackendTLS + if tls.Root != tc.roots || tls.Cert != "/run/podidentity.podcert.ate.dev/credential-bundle.pem" || tls.Key != tls.Cert { + t.Fatalf("incorrect provider mTLS: %+v", tls) + } + } + } + } + } + } + if providers["HTTP"] != 1 || providers["HTTPS"] != 1 { + t.Fatalf("providers=%v, want one per HTTP/HTTPS route", providers) + } + if mitmMounts != 1 || mitmVolumes != 1 || passthroughListeners != 0 { + t.Fatalf("MITM mounts=%d volumes=%d passthrough listeners=%d", mitmMounts, mitmVolumes, passthroughListeners) + } + }) + } +} diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz.go b/cmd/credential-provider/kubernetes-secrets/nsauthz.go index d5ffe13348..c648220898 100644 --- a/cmd/credential-provider/kubernetes-secrets/nsauthz.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz.go @@ -18,6 +18,9 @@ import ( "fmt" "os" + "github.com/agent-substrate/substrate/internal/resources" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/yaml" ) @@ -48,7 +51,7 @@ func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { return nil, fmt.Errorf("reading namespace policy file %q: %w", path, err) } var file namespacePolicyFile - if err := yaml.Unmarshal(data, &file); err != nil { + if err := yaml.UnmarshalStrict(data, &file); err != nil { return nil, fmt.Errorf("parsing namespace policy file %q: %w", path, err) } return newNamespaceAuthorizer(file) @@ -59,8 +62,8 @@ func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, error) { allowed := make(map[string]map[string]struct{}) for i, p := range file.Policies { - if p.Atespace == "" { - return nil, fmt.Errorf("namespace policy %d: atespace is required", i) + if !resources.IsValidResourceName(p.Atespace) { + return nil, fmt.Errorf("namespace policy %d: valid atespace is required", i) } set := allowed[p.Atespace] if set == nil { @@ -68,6 +71,9 @@ func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, err allowed[p.Atespace] = set } for _, ns := range p.AllowedNamespaces { + if len(validation.IsDNS1123Label(ns)) != 0 { + return nil, fmt.Errorf("namespace policy %d: invalid namespace %q", i, ns) + } set[ns] = struct{}{} } } @@ -78,6 +84,9 @@ func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, err // deny: an atespace absent from the mapping, or a namespace not in its list, is // refused. func (a *NamespaceAuthorizer) Allowed(atespace, namespace string) bool { + if a == nil { + return false + } set, ok := a.allowed[atespace] if !ok { return false diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index 996f4f1adb..ef072c9916 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -435,7 +435,8 @@ for a worked example. **Substrate exports one thing over OTLP: the actor lifecycle events**, from ateapi, through `serverboot.InitLogging`. They are off unless -`OTEL_LOGS_EXPORTER=otlp` is set, which only the kind overlay does today. See +`OTEL_LOGS_EXPORTER=otlp` is set. The kind overlay sets it, and the Helm chart +sets it from `otel.logs.enabled` once `otel.endpoint` resolves. See [the same records over OTLP](../../observability.md#the-same-records-over-otlp). Everything else is stdout. `serverboot.InitLogger` writes structured JSON there, diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md new file mode 100644 index 0000000000..aba95669fd --- /dev/null +++ b/docs/kubernetes-credential-provider.md @@ -0,0 +1,102 @@ +# Kubernetes credential provider + +The `k8s-credential-provider` Deployment follows the provider from +[upstream](https://github.com/agent-substrate/substrate/pull/1335). It serves +`CredentialProvider.FetchSecret` at `k8s-credential-provider.ate-system.svc:50051` +with its own ServiceAccount and projected serving certificate. AGW calls it +directly over mTLS to inject credentials into HTTP and intercepted HTTPS requests. + +`ate-secret://k8s.io/default/team-a-secrets/example-api/token` resolves the `token` +entry in that Kubernetes Secret. The `default` locator and an explicit key are required. +The provider reads Kubernetes on every fetch and never persists or logs values. +AGW caches successful credentials per actor and URI for five minutes, so rotation +can take that long to reach injected requests. + +Each request requires a trusted injector certificate with the configured SPIFFE +identity, an explicit atespace-to-namespace grant for the attested actor, and +Kubernetes `get` permission for the provider's ServiceAccount. Both installers +include the upstream get-only Secret ClusterRole and bind it to that ServiceAccount. +The provider can read Secrets across namespaces; its namespace policy controls +which namespaces each actor may use. Empty policies deny all requests. + +## Configure the provider + +Keep the pinned `images.agentgateway` image. It includes the +[protocol update](https://github.com/agentgateway/agentgateway/pull/3524) from +[this build](https://github.com/agentgateway/agentgateway/actions/runs/35238449333) +and implements the current [FetchSecret contract](../pkg/proto/credproviderpb/credprovider.proto). + +Create the MITM CA Secret using the existing installation tooling: + +```sh +hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret +``` + +The gateway needs `egress-mitm-ca-pool` with `tls.crt` and `tls.key` in its namespace. +Actors making HTTPS requests must trust this CA; see the +[MITM trust bundle guide](egress-trust-bundle.md). + +For Helm, add these values to your release configuration: + +```yaml +credentialProvider: + namespacePolicies: + - atespace: team-a + allowedNamespaces: [team-a-secrets] +``` + +The Helm chart always deploys the provider and configures AGW's HTTP route and +HTTPS interception route. Namespace grants default to an empty list. +Policy changes roll the provider's Pods. Resource names and the injector identity +follow the release: release `demo` in namespace `platform` uses ServiceAccount +`demo-k8s-credential-provider`, endpoint +`demo-k8s-credential-provider.platform.svc:50051`, and injector identity +`spiffe://cluster.local/ns/platform/sa/demo-atenet-egress`. + +HTTPS uses a dynamic backend: AGW selects the destination from the request and +validates its certificate using the system CA roots (`backendTLS: {}`). Public +APIs such as OpenAI and Anthropic need no per-backend certificates. The single +MITM CA lets AGW generate actor-facing certificates as needed. HTTP also travels +through the authenticated CONNECT tunnel, then leaves AGW over plaintext HTTP. + +For the manifest installer, set your grants in +`manifests/egress-credential-injection/namespace-policy.yaml`, then deploy: + +```sh +kubectl kustomize manifests/egress-credential-injection | ko apply -f - +hack/install-ate.sh --deploy-atenet \ + --atenet-dataplane=agentgateway --experimental-use-sdsmint +``` + +The policy file uses `policies:` with the same list of grants as the Helm values. +Its generated ConfigMap name changes with the policy, rolling the provider on +reapplication. Direct policy ConfigMap edits require a rollout restart. Client CA +bundles and serving certificates reload automatically for new TLS connections. + +## Configure injection + +Create the Secret and set an actor's egress policy header injection to use credential URI +`ate-secret://k8s.io/default/team-a-secrets/example-api/token`, header +`authorization`, and prefix `Bearer `. Namespace grants alone do not create an +egress policy. No ext_proc injector is needed. + +## Tests + +The Helm PR workflow installs the provider and MITM gateway from the start and +runs `internal/e2e/suites/credentials` alongside the standard suites with real actors, +Secrets, chart-managed RBAC, AGW, and the deployed provider. It checks +the exact injected token, an unauthenticated-origin control, namespace-policy +denial and cache isolation between atespaces. The local origin serves HTTP; +the suite uses the installed gateway configuration without modifying ConfigMaps. + +Include `-f internal/e2e/suites/credentials/values.yaml` in the initial Helm +installation to grant the test atespace access. After deploying the standard +MITM egress fixtures, run the suites together: + +```sh +E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 E2E_EGRESS_MITM=1 \ + hack/run-e2e-kind.sh -v -args --no-color +``` + +The credential suite tests HTTP injection. The existing MITM suite checks HTTPS +interception and actor trust against a public HTTPS origin using the same install. diff --git a/docs/observability.md b/docs/observability.md index 2ddc37be49..b8c512cd04 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -178,7 +178,7 @@ The counter carries the same reason but no actor identity, so this record is the #### The same records over OTLP -Both records also go out as OTLP log events, so a collector reads them without knowing substrate's stdout envelope. Set `OTEL_LOGS_EXPORTER=otlp` to turn it on; unset means `none`, which is what every environment but kind uses today. Only ateapi has a LoggerProvider — a worker pod cannot export a log record at all yet, because [the ateom relay](#the-ateom-otlp-relay) carries traces and metrics only. +Both records also go out as OTLP log events, so a collector reads them without knowing substrate's stdout envelope. Set `OTEL_LOGS_EXPORTER=otlp` to turn it on; unset means `none`. The kind overlay sets it, and a chart install gets it from `otel.logs.enabled` once `otel.endpoint` resolves. Only ateapi has a LoggerProvider — a worker pod cannot export a log record at all yet, because [the ateom relay](#the-ateom-otlp-relay) carries traces and metrics only. Two `event.name` values, which is the OTLP LogRecord's own field rather than an attribute: @@ -405,7 +405,7 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Path | service → in-cluster `opentelemetry-collector` | service → Google Managed Prometheus (GMP) | | Metrics | collector Prometheus exporter on `:8889` | Google Cloud Monitoring | | Traces | Jaeger UI | Google Cloud Trace | -| Logs | pod stdout; ateapi's [actor lifecycle events](#the-same-records-over-otlp) also to the collector's `debug` exporter | pod stdout. No OTLP logs: `OTEL_LOGS_EXPORTER` is unset | +| Logs | pod stdout; ateapi's [actor lifecycle events](#the-same-records-over-otlp) also to the collector's `debug` exporter | pod stdout. No OTLP logs unless `OTEL_LOGS_EXPORTER` is set | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | > In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the ateom worker pods it creates, so all component telemetry lands locally. diff --git a/go.mod b/go.mod index e03fe4cca8..5d7d19d807 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spiffe/go-spiffe/v2 v2.7.0 + github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/vishvananda/netlink v1.3.1 @@ -220,7 +221,6 @@ require ( github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/viper v1.20.1 // indirect - github.com/stretchr/testify v1.12.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 0000000000..baa22fa517 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index c8b287b2b1..e771c9cd54 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -174,6 +174,7 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." + run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 0000000000..e75108a1cb --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" +PRESERVED_FILES=( + ate-api-server.yaml + ate-controller.yaml + ate-otel-config.yaml + ate-system-namespace.yaml + atelet.yaml + atenet-egress.yaml + atenet-egress-with-sdsmint.yaml + atenet-router.yaml + atenet-router-monitoring.yaml + # The provider's upstream manifest lives in manifests/egress-credential-injection. + k8s-credential-provider.yaml + pod-certificate-controller.yaml + postgres.yaml + sandboxconfig-gvisor.yaml + sandboxconfig-validation.yaml +) + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com \ + --set image.repository=agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + if src == "namespace.yaml": + src = "ate-system-namespace.yaml" + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + for file in "${PRESERVED_FILES[@]}"; do + rm -f "${CHECK_TMP}/current/${file}" "${TMP_DIR}/out/${file}" + done + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ + ! -name 'ate-api-server.yaml' \ + ! -name 'ate-controller.yaml' \ + ! -name 'ate-otel-config.yaml' \ + ! -name 'ate-system-namespace.yaml' \ + ! -name 'atelet.yaml' \ + ! -name 'atenet-egress.yaml' \ + ! -name 'atenet-egress-with-sdsmint.yaml' \ + ! -name 'atenet-router.yaml' \ + ! -name 'atenet-router-monitoring.yaml' \ + ! -name 'pod-certificate-controller.yaml' \ + ! -name 'postgres.yaml' \ + ! -name 'sandboxconfig-gvisor.yaml' \ + ! -name 'sandboxconfig-validation.yaml' \ + -delete +for file in "${PRESERVED_FILES[@]}"; do + rm -f "${TMP_DIR}/out/${file}" +done +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 27c66dffe1..73c3a4bb91 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -55,10 +55,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -if [[ $# -gt 0 ]]; then - echo "Error: unknown argument $1" >&2 - exit 1 -fi +SKIP_CONTROL_PLANE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; + *) + echo "Error: unknown argument $1" >&2 + exit 1 + ;; + esac + shift +done if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is required (set it in .ate-dev-env.sh for GKE," >&2 @@ -75,13 +83,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 99e24d731b..2f2dbfde6a 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="_LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 0000000000..dc3ef2bdaf --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"ate.dev_*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"ate.dev_*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 31763ccf62..00de8bd61c 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -24,6 +24,7 @@ import ( "strings" "sync" + "github.com/agent-substrate/substrate/internal/installdefaults" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -184,7 +185,7 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext, tokenFile // TODO: Should we special-case a LoadBalancer "api" Service and dial its // address directly instead of port-forwarding? - localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, "ate-system", "api", 443) + localPort, stopForward, err := portforward.ServicePortForward(ctx, config, clientset, installdefaults.SystemNamespace, installdefaults.APIServiceName, 443) if err != nil { return nil, err } @@ -273,7 +274,7 @@ func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset, ExpirationSeconds: &expirationSeconds, }, } - token, err := clientset.CoreV1().ServiceAccounts("ate-system").CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) + token, err := clientset.CoreV1().ServiceAccounts(installdefaults.SystemNamespace).CreateToken(ctx, "ate-client", tokenRequest, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to request ateapi bearer token: %w", err) } diff --git a/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go index 542c05d9ac..d24d2f3a72 100644 --- a/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -162,6 +163,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -174,8 +176,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -189,7 +192,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -220,3 +223,16 @@ func ParsePool(path string) (*x509.CertPool, error) { } return pool, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go index 4b5e818d83..6685604633 100644 --- a/internal/credbundle/credbundle_test.go +++ b/internal/credbundle/credbundle_test.go @@ -58,13 +58,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t, 1) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index 956ca53045..e793ac63b2 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -147,6 +147,16 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, t.Fatalf("fixture %s declares templates in different atespaces (%q and %q)", manifests.Template, atespace, got) } } + t.Cleanup(func() { + // Remove workers before the namespace so its controller does not wait + // on their one-hour termination grace estimate after the Pods exit. + delArgs := []string{"delete", "workerpools", "--all", "--namespace=" + atespace, + "--ignore-not-found", "--cascade=foreground", "--timeout=2m"} + if KubeContext != "" { + delArgs = append([]string{"--context=" + KubeContext}, delArgs...) + } + RunCmd(t, "kubectl", delArgs...) + }) if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: atespace}}}); err != nil && status.Code(err) != codes.AlreadyExists { t.Fatalf("failed to create atespace %q: %v", atespace, err) diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 0ae303baa8..5eaf72806d 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -17,19 +17,17 @@ package main import ( "log" "net/http" + "os" "time" "github.com/spf13/cobra" ) // newHTTPCmd is a plain HTTP/1.1 origin an Actor's egress lands on. It exists so -// a test can assert the destination port is recovered from SO_ORIGINAL_DST -// rather than defaulted from the URL scheme: the actor fetches its /healthz on a -// non-standard port, and the gateway's access log is expected to carry that -// port. There is nothing to serve beyond readiness, so /healthz is all it -// answers. +// a test can assert the destination port is recovered from SO_ORIGINAL_DST. +// It can also verify an injected Authorization header against a mounted token. func newHTTPCmd() *cobra.Command { - var listenAddress string + var listenAddress, authorizationFile string cmd := &cobra.Command{ Use: "http", Short: "Serve a plain HTTP/1.1 origin answering /healthz.", @@ -39,6 +37,9 @@ func newHTTPCmd() *cobra.Command { mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + if authorizationFile != "" { + mux.HandleFunc("/credential", credentialHandler(authorizationFile)) + } server := &http.Server{ Addr: listenAddress, @@ -51,5 +52,21 @@ func newHTTPCmd() *cobra.Command { }, } cmd.Flags().StringVar(&listenAddress, "listen", ":8080", "Address the HTTP origin listens on.") + cmd.Flags().StringVar(&authorizationFile, "authorization-file", "", "Enable /credential, requiring a Bearer token matching this file.") return cmd } + +func credentialHandler(path string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, err := os.ReadFile(path) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + if len(token) == 0 || r.Header.Get("Authorization") != "Bearer "+string(token) { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusNoContent) + } +} diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go new file mode 100644 index 0000000000..b42c292677 --- /dev/null +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestCredentialHandler(t *testing.T) { + path := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(path, []byte("expected-token"), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + header string + status int + }{ + {"", http.StatusUnauthorized}, + {"Bearer wrong-token", http.StatusUnauthorized}, + {"Bearer expected-token", http.StatusNoContent}, + } { + req := httptest.NewRequest(http.MethodGet, "/credential", nil) + req.Header.Set("Authorization", tc.header) + resp := httptest.NewRecorder() + credentialHandler(path)(resp, req) + if resp.Code != tc.status || resp.Body.Len() != 0 { + t.Errorf("header %q: status=%d body=%q, want status=%d and no body", tc.header, resp.Code, resp.Body.String(), tc.status) + } + } + resp := httptest.NewRecorder() + credentialHandler(path+"-missing")(resp, httptest.NewRequest(http.MethodGet, "/credential", nil)) + if resp.Code != http.StatusInternalServerError { + t.Fatalf("unreadable credential file: status=%d, want 500", resp.Code) + } +} diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go new file mode 100644 index 0000000000..2e39fc128e --- /dev/null +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credentials + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// TestKubernetesCredentialInjection uses the Helm values in values.yaml, real +// actors, and a local HTTP origin with the installed gateway configuration and RBAC. +func TestKubernetesCredentialInjection(t *testing.T) { + if os.Getenv("E2E_CREDENTIAL_PROVIDER") == "" { + t.Skip("requires credential E2E namespace grants and E2E_CREDENTIAL_PROVIDER=1") + } + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + require.NoError(t, err) + ctx := t.Context() + clients := e2e.GetClients() + namespace, template := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials") + deniedAtespace, deniedTemplate := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials-denied") + otherNamespace := e2e.CreateNamespace(t).Name + + for _, secret := range []struct{ namespace, name string }{ + {namespace, "allowed"}, {otherNamespace, "allowed"}, + } { + _, err := clients.K8s.CoreV1().Secrets(secret.namespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secret.name}, + Data: map[string][]byte{"token": []byte("e2e-credential-token")}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + + e2e.DeployServerPod(t, ctx, e2e.ServerPod{ + Name: "credential-origin", Namespace: namespace, + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--authorization-file=/run/token/token"}, + Port: 80, TargetPort: 8080, + Volumes: []corev1.Volume{ + {Name: "token", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "allowed"}}}, + }, + VolumeMounts: []corev1.VolumeMount{{Name: "token", MountPath: "/run/token", ReadOnly: true}}, + }) + host := "credential-origin." + namespace + ".svc" + router, err := e2e.NewRouterClient(ctx) + require.NoError(t, err) + t.Cleanup(router.Close) + // Keep the successful actor alive through the cache-isolation check. + suite := t + for _, tc := range []struct { + name, secretNamespace, secret, want string + }{ + {"without-injection", "", "", "401"}, + {"allowed", namespace, "allowed", "204"}, + {"atespace-denied", namespace, "allowed", "403"}, + {"namespace-denied", otherNamespace, "allowed", "403"}, + } { + t.Run(tc.name, func(t *testing.T) { + atespace, actorTemplate := namespace, template + actorName := tc.name + if tc.name == "atespace-denied" { + // Use the already-fetched URI from an ungranted atespace so an + // incorrectly shared gateway cache cannot bypass authorization. + atespace, actorTemplate = deniedAtespace, deniedTemplate + actorName = "allowed" + } + actor := &ateapipb.ObjectRef{Atespace: atespace, Name: actorName} + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actor}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actor}) + _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: actorName}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: actorTemplate.GetMetadata().GetName()}, + }}) + require.NoError(t, err) + cleanupTest := t + if tc.name == "allowed" { + cleanupTest = suite + } + cleanupTest.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: actor}) + _, err := clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: actor}) + if err != nil { + cleanupTest.Errorf("delete actor %s/%s: %v", atespace, actorName, err) + } + }) + rule := e2e.EgressAllowHostnames(host) + if tc.secret != "" { + rule.Hostnames.Effects = &ateapipb.EgressRuleEffects{InjectStaticHeaders: []*ateapipb.CredentialHeaderInjection{{ + Header: "authorization", Prefix: "Bearer ", + CredentialUri: fmt.Sprintf("ate-secret://k8s.io/default/%s/%s/token", tc.secretNamespace, tc.secret), + }}} + } + e2e.EnsureEgressPolicy(t, ctx, clients, actor, rule) + _, err = clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actor}) + require.NoError(t, err) + path := "/fetch?roots=system&url=" + url.QueryEscape("http://"+host+"/credential") + // Route discovery is asynchronous. Retries require the precise status; + // transport errors never pass. + deadline := time.Now().Add(90 * time.Second) + for { + resp, err := router.Get(ctx, resources.ActorRef{Atespace: atespace, Name: actorName}, path) + var body []byte + if err == nil { + body, err = io.ReadAll(resp.Body) + resp.Body.Close() + var result struct{ Status, Error string } + if err == nil && resp.StatusCode == http.StatusOK && json.Unmarshal(body, &result) == nil && result.Error == "" && result.Status == tc.want { + break + } + } + if time.Now().After(deadline) { + t.Fatalf("want origin status %s; last response: %s; error: %v", tc.want, body, err) + } + time.Sleep(2 * time.Second) + } + }) + } +} diff --git a/internal/e2e/suites/credentials/testmain_test.go b/internal/e2e/suites/credentials/testmain_test.go new file mode 100644 index 0000000000..6b8549a17a --- /dev/null +++ b/internal/e2e/suites/credentials/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credentials + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/credentials/values.yaml b/internal/e2e/suites/credentials/values.yaml new file mode 100644 index 0000000000..27ff95125a --- /dev/null +++ b/internal/e2e/suites/credentials/values.yaml @@ -0,0 +1,18 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +credentialProvider: + namespacePolicies: + - atespace: ate-e2e-probe-credentials + allowedNamespaces: [ate-e2e-probe-credentials] diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 99a9cfd485..eecdf7bafc 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "net/http" + "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -706,7 +709,7 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory if !strings.Contains(resp, memoryCounterPrefix+fmt.Sprintf("%d", wantMemory)) { t.Errorf("[%s] expected memory count %d, got response: %s", stage, wantMemory, resp) } - if !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { + if wantFile >= 0 && !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { t.Errorf("[%s] expected file count %d, got response: %s", stage, wantFile, resp) } } @@ -730,24 +733,14 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj }) }() - listResp, err := clients.SubstrateAPI.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: demoAtespace}) + getResp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + }) if err != nil { - t.Fatalf("ListActors RPC failed: %v", err) - } - - var myActors []*ateapipb.Actor - for _, actor := range listResp.GetActors() { - if actor.GetActorTemplate().GetName() == at.GetMetadata().GetName() && actor.GetMetadata().GetName() == actorName { - myActors = append(myActors, actor) - } + t.Fatalf("GetActor RPC failed: %v", err) } - // Check that we have our Actor created. - if len(myActors) != 1 { - t.Fatalf("expected actor %s from template %s, got %d actors: %v", actorName, at.GetMetadata().GetName(), len(myActors), myActors) - } - - actor := myActors[0] + actor := getResp if actor.GetMetadata().GetName() != actorName { t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName()) } @@ -758,8 +751,7 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State) } - t.Logf("Successfully queried Substrate API. Found %d active actors total, %d from our template %s.", - len(listResp.GetActors()), len(myActors), at.GetMetadata().GetName()) + t.Logf("Successfully queried Substrate API. Found actor %s in namespace %s.", actorName, nsObj.Name) return nil } @@ -786,13 +778,13 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) - // Pausing the actor t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ @@ -811,11 +803,12 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after pause", 2, -1) + } else { + validateCounterResponse(t, resp, "after pause", 2, 2) } - validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -865,11 +858,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -889,11 +883,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after suspend", 2, -1) + } else { + validateCounterResponse(t, resp, "after suspend", 2, 2) } - validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -1323,6 +1318,55 @@ func waitForActorStateWithTimeout(ctx context.Context, t *testing.T, clients *e2 t.Fatalf("timed out waiting for actor %q to reach state %v", actorName, expectedState) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, actorRef resources.ActorRef, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, actorRef) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorRef.Name, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorRef.Name, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + +func isMicroVMEnvironment() bool { + return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" +} + func callActor(t *testing.T, actorRef resources.ActorRef) (string, error) { return callActorPath(t, actorRef, "POST", "/") } diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index bf46ffa2c0..d2bae4ae18 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -278,11 +278,17 @@ func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Client func whoami(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) whoamiResponse { t.Helper() - out, err := tryWhoami(ctx, rc, id) - if err != nil { - t.Fatal(err) + deadline := time.Now().Add(30 * time.Second) + for { + out, err := tryWhoami(ctx, rc, id) + if err == nil { + return out + } + if time.Now().After(deadline) { + t.Fatal(err) + } + time.Sleep(time.Second) } - return out } // tryWhoami is whoami returning the error instead of failing the test. diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index e0a0302347..9a7d57a99c 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -652,24 +652,29 @@ func (s *Store) pull(ctx context.Context, parsedRef name.Reference, digest v1.Ha // collapsing concurrent requests for the same layer across images. func (s *Store) ensureLayer(ctx context.Context, diffID v1.Hash, layer v1.Layer) (string, error) { dir := s.layerDir(diffID) - _, err, _ := s.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { - if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { - // Refresh the dir mtime inside the flight: retireLayer re-checks - // the mtime in this same flight, so a layer reused here can - // never be renamed away between this stat and the image record - // that will re-reference it. - now := time.Now() - if err := os.Chtimes(dir, now, now); err != nil { - slog.WarnContext(ctx, "Failed to refresh layer mtime on reuse", slog.String("diffid", diffID.String()), slog.Any("err", err)) + for { + result, err, _ := s.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { + if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { + // Refresh the dir mtime inside the flight: retireLayer re-checks + // the mtime in this same flight, so a layer reused here can + // never be renamed away between this stat and the image record + // that will re-reference it. + now := time.Now() + if err := os.Chtimes(dir, now, now); err != nil { + slog.WarnContext(ctx, "Failed to refresh layer mtime on reuse", slog.String("diffid", diffID.String()), slog.Any("err", err)) + } + return dir, nil } - return nil, nil + return dir, s.unpackLayerToPool(ctx, diffID, layer) + }) + if err != nil { + return "", err + } + // A joined retirement returns nil, so the layer still needs a pull. + if result != nil { + return dir, nil } - return nil, s.unpackLayerToPool(ctx, diffID, layer) - }) - if err != nil { - return "", err } - return dir, nil } // unpackLayerToPool streams the layer (download → decompress → untar) into a diff --git a/internal/imagecache/retire_test.go b/internal/imagecache/retire_test.go index fe45c5a3cc..2abf3efa28 100644 --- a/internal/imagecache/retire_test.go +++ b/internal/imagecache/retire_test.go @@ -22,6 +22,7 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -201,3 +202,40 @@ func TestRetireLayerVsEnsureImageRace(t *testing.T) { t.Errorf("final layer dir missing: %v", err) } } + +func TestEnsureLayerJoinsRetirement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + store := newTestStore(t) + layer := layerFromEntries(t, []tarEntry{ + {name: "f", typeflag: tar.TypeReg, mode: 0o644, body: "hi"}, + }) + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + + // Hold a retirement flight while the pull joins it. + release := make(chan struct{}) + go func() { + _, _, _ = store.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { + <-release + return nil, nil + }) + }() + synctest.Wait() + var dir string + go func() { + dir, err = store.ensureLayer(context.Background(), diffID, layer) + }() + synctest.Wait() + close(release) + synctest.Wait() + + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, layerFSDirName, "f")); err != nil { + t.Fatalf("layer was not unpacked after retirement: %v", err) + } + }) +} diff --git a/internal/installdefaults/installdefaults.go b/internal/installdefaults/installdefaults.go new file mode 100644 index 0000000000..6d6475bc32 --- /dev/null +++ b/internal/installdefaults/installdefaults.go @@ -0,0 +1,43 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package installdefaults holds the default namespace and Service names +// that match the canonical install layout in manifests/ate-install/. +// Binaries use these as flag defaults; deployments that diverge from +// the canonical layout pass actual values via the corresponding flags. +package installdefaults + +import "os" + +const ( + // SystemNamespace is the namespace where substrate's control-plane + // components and the atelet DaemonSet run. + SystemNamespace = "ate-system" + // APIServiceName is the Service name of ate-api-server. + APIServiceName = "api" + + // PodNamespaceEnv is the conventional env var name for the namespace + // a pod is running in, exposed via Kubernetes' downward API. + PodNamespaceEnv = "POD_NAMESPACE" +) + +// NamespaceFromPodEnv returns the namespace from the PodNamespaceEnv env +// var when set (typically populated via Kubernetes' downward API), and +// falls back to SystemNamespace for non-k8s invocations (tests, local dev). +func NamespaceFromPodEnv() string { + if ns := os.Getenv(PodNamespaceEnv); ns != "" { + return ns + } + return SystemNamespace +} diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index ad71177441..aaa8edce05 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -391,6 +391,11 @@ func TestCAPoolCache_HitAndFileChange(t *testing.T) { // Modify the file. writeFile(t, caPath, ca.certPEM()) + // Advance mtime explicitly; consecutive writes can share a filesystem tick. + modified := cache.fi.ModTime().Add(time.Second) + if err := os.Chtimes(caPath, modified, modified); err != nil { + t.Fatal(err) + } // 3rd call should detect file change and return a newly parsed pool. pool3, err := cache.getCertPool() diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..b49cff6e1e --- /dev/null +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-server-envvars + namespace: ate-system +data: + ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + ATE_API_POSTGRES_SCHEMA: "public" diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 0000000000..e59bd53f8f --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml index b90274819d..6b72ca6de0 100644 --- a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml @@ -72,6 +72,15 @@ patches: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: k8s-credential-provider.ate-system.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem - protocol: HTTP routes: - backends: @@ -85,6 +94,15 @@ patches: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: k8s-credential-provider.ate-system.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem - protocol: TCP tcpRoutes: - backends: diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 7fee9bbc1a..a7011a504f 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -58,6 +58,32 @@ data: insecureHost: true routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + # AgentGateway only uses atunnel's CONNECT listener. + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors gateways: - http @@ -77,6 +103,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 # Terminate client CONNECT before internal HTTP routing. binds: @@ -97,6 +126,28 @@ data: listeners: - protocol: HTTP routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors-tunneled matches: - path: @@ -112,6 +163,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 --- apiVersion: v1 diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index fe062e4447..cdff87553f 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e args: - -f - /etc/agentgateway/config.yaml @@ -118,7 +118,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e args: - -f - /etc/agentgateway/config.yaml diff --git a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml index e482ab415e..80cbd132b8 100644 --- a/manifests/ate-install/kind/otel-collector.yaml +++ b/manifests/ate-install/kind/otel-collector.yaml @@ -107,7 +107,6 @@ data: receivers: [otlp] processors: [batch] exporters: [prometheus, debug] - # Every ate event, plus any workload that sends OTLP logs. debug is the # only exporter: kind has no log store, and the e2e read-back goes # through the count connector below. Never put a sampler or a filter diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 0000000000..65e967c2a6 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,130 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ate-controller + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml new file mode 100644 index 0000000000..d6be308128 --- /dev/null +++ b/manifests/ate-install/rustfs.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: ate-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: rustfs + namespace: ate-system +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustfs + namespace: ate-system +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: "rustfsadmin" + - name: RUSTFS_SECRET_KEY + value: "rustfsadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: rustfs-data +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rustfs-bucket-init + namespace: ate-system +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots already exists" + exit 0 + fi + if aws s3api create-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 5144d818fe..52eae32afe 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -22,9 +22,7 @@ metadata: name: k8s-credential-provider namespace: ate-system --- -# POC scope note: this grants read on Secrets cluster-wide so a policy can name a -# Secret in any namespace. A production deployment would scope this to the -# namespaces a provider instance is allowed to serve. +# The provider checks the actor's atespace-to-namespace grant before reading. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/manifests/egress-credential-injection/kustomization.yaml b/manifests/egress-credential-injection/kustomization.yaml new file mode 100644 index 0000000000..19bf054fef --- /dev/null +++ b/manifests/egress-credential-injection/kustomization.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- k8s-credential-provider.yaml + +configMapGenerator: +- name: k8s-credential-provider-namespace-policy + namespace: ate-system + files: + - namespace-policy.yaml diff --git a/manifests/egress-credential-injection/namespace-policy.yaml b/manifests/egress-credential-injection/namespace-policy.yaml index d98d1ae76b..78747a0d26 100644 --- a/manifests/egress-credential-injection/namespace-policy.yaml +++ b/manifests/egress-credential-injection/namespace-policy.yaml @@ -12,23 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The atespace→namespace authorization policy k8s-credential-provider enforces: an actor's -# atespace may only resolve Secrets whose URI namespace is listed for it here. -# Enforcement is default-deny — an atespace absent from this file resolves -# nothing. -# -# TODO: k8s-credential-provider loads this once at startup, so editing this ConfigMap -# requires restarting the k8s-credential-provider Deployment. Make it reload dynamically. -apiVersion: v1 -kind: ConfigMap -metadata: - name: k8s-credential-provider-namespace-policy - namespace: ate-system -data: - namespace-policy.yaml: | - # atespace "team-a" may resolve secrets in namespace "ns1" (matches the - # sample policy's ate-secret://k8s.io/default/ns1/example-api/token). - policies: - - atespace: team-a - allowedNamespaces: - - ns1 +# Default-deny atespace-to-namespace grants. Secret RBAC is configured separately. +policies: []