From 2a29fd1e2ae68ebf2d395f3b0dfc9ffbb3b62d36 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 09:14:50 +0200 Subject: [PATCH 01/60] feat(ref-arch): draft STACKIT AI reference architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Draft scaffold for discussion β€” README and diagrams only. Centers on enterprise-grade sovereign model serving: OpenWebUI as the user-facing interface, LiteLLM as the governance choke point (virtual keys, budgets, model allow-lists), Langfuse for tracing and usage attribution, and STACKIT AI Model Serving as the sovereign backend. Adds a second diagram for the pluggable-model (bring your own model) variant so the STACKIT-specific vs. generic AI platform scoping question can be discussed against a visual. FlowiseAI and RAGFlow from the original demo stack are intentionally left out to keep the focus on serving, observability and governance. Also scaffolds modules/stackit/model-serving as a minimal first cut around stackit_modelserving_token, and extends the diagram emoji vocabulary with user/chat/gateway/observability concepts. Co-Authored-By: Claude Opus 5 (1M context) --- .agents/references/diagrams.md | 6 +- .../model-serving/buildingblock/README.md | 51 ++++++ .../model-serving/buildingblock/main.tf | 7 + .../model-serving/buildingblock/outputs.tf | 15 ++ .../model-serving/buildingblock/provider.tf | 4 + .../model-serving/buildingblock/variables.tf | 32 ++++ .../model-serving/buildingblock/versions.tf | 10 + reference-architectures/stackit-ai/README.md | 130 +++++++++++++ .../stackit-ai/stackit-ai-pluggable.dot | 62 +++++++ .../stackit-ai/stackit-ai-pluggable.svg | 131 ++++++++++++++ .../stackit-ai/stackit-ai.dot | 74 ++++++++ .../stackit-ai/stackit-ai.svg | 171 ++++++++++++++++++ 12 files changed, 692 insertions(+), 1 deletion(-) create mode 100644 modules/stackit/model-serving/buildingblock/README.md create mode 100644 modules/stackit/model-serving/buildingblock/main.tf create mode 100644 modules/stackit/model-serving/buildingblock/outputs.tf create mode 100644 modules/stackit/model-serving/buildingblock/provider.tf create mode 100644 modules/stackit/model-serving/buildingblock/variables.tf create mode 100644 modules/stackit/model-serving/buildingblock/versions.tf create mode 100644 reference-architectures/stackit-ai/README.md create mode 100644 reference-architectures/stackit-ai/stackit-ai-pluggable.dot create mode 100644 reference-architectures/stackit-ai/stackit-ai-pluggable.svg create mode 100644 reference-architectures/stackit-ai/stackit-ai.dot create mode 100644 reference-architectures/stackit-ai/stackit-ai.svg diff --git a/.agents/references/diagrams.md b/.agents/references/diagrams.md index 87abb5d1..37237fae 100644 --- a/.agents/references/diagrams.md +++ b/.agents/references/diagrams.md @@ -93,6 +93,10 @@ in one diagram to a box in another without reading the label. | πŸ”Œ | network, subnet | | ☸️ | Kubernetes cluster | | πŸ—„οΈ | container registry | +| πŸ‘€ | end user, application team acting as a user | +| πŸ’¬ | user-facing chat interface | +| πŸšͺ | gateway, abstraction layer | +| πŸ“ˆ | observability, tracing, evaluation | | βš™οΈ | CI/CD wiring | | πŸ”€ | git repository | | 🧠 | model serving / AI API | @@ -105,7 +109,7 @@ a concept already listed. Copy the emoji straight from this table β€” several of them carry a **U+FE0F variation selector** that is invisible in the source but decides whether the glyph renders in colour. `☸ βš™ πŸ—„ πŸ—‚ πŸ›°` default to -*text* presentation and come out as small monochrome symbols without it; `🏒 πŸ“ πŸ”‘ 🌐 πŸ”Œ πŸ”€ 🧠 πŸ›¬ πŸ“¦` +*text* presentation and come out as small monochrome symbols without it; `🏒 πŸ“ πŸ”‘ 🌐 πŸ”Œ πŸ”€ 🧠 πŸ›¬ πŸ“¦ πŸ‘€ πŸ’¬ πŸšͺ πŸ“ˆ` are colour by default. If an icon renders monochrome in the SVG, it is missing the selector. The **same emoji with a different fill** is deliberate and useful: a green `πŸ”€` repo inside the diff --git a/modules/stackit/model-serving/buildingblock/README.md b/modules/stackit/model-serving/buildingblock/README.md new file mode 100644 index 00000000..854d170b --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/README.md @@ -0,0 +1,51 @@ +--- +name: STACKIT AI Model Serving Access +supportedPlatforms: + - stackit +description: Issues a scoped STACKIT AI Model Serving API token so a tenant can call the sovereign LLM API. +--- + +# STACKIT AI Model Serving Access Building Block + + + +This building block issues a STACKIT AI Model Serving API token scoped to a tenant's STACKIT project. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [stackit](#requirement\_stackit) | >= 0.88.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_modelserving_token.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/modelserving_token) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [project\_id](#input\_project\_id) | STACKIT project ID the Model Serving token is issued in. | `string` | n/a | yes | +| [region](#input\_region) | STACKIT region for the Model Serving token. Defaults to the provider region. | `string` | `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Email of the STACKIT service account used to issue the Model Serving token. | `string` | n/a | yes | +| [token\_description](#input\_token\_description) | Description shown on the Model Serving token. | `string` | `"Managed by meshStack."` | no | +| [token\_name](#input\_token\_name) | Display name of the Model Serving token. | `string` | n/a | yes | +| [ttl\_duration](#input\_ttl\_duration) | Lifetime of the Model Serving token, e.g. '90d'. | `string` | `"90d"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [token](#output\_token) | The STACKIT Model Serving API token. Shown only on creation. | +| [token\_id](#output\_token\_id) | ID of the Model Serving token. | +| [valid\_until](#output\_valid\_until) | Expiry timestamp of the Model Serving token. | + diff --git a/modules/stackit/model-serving/buildingblock/main.tf b/modules/stackit/model-serving/buildingblock/main.tf new file mode 100644 index 00000000..91d6c4a1 --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/main.tf @@ -0,0 +1,7 @@ +resource "stackit_modelserving_token" "this" { + project_id = var.project_id + region = var.region + name = var.token_name + description = var.token_description + ttl_duration = var.ttl_duration +} diff --git a/modules/stackit/model-serving/buildingblock/outputs.tf b/modules/stackit/model-serving/buildingblock/outputs.tf new file mode 100644 index 00000000..7fd10643 --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/outputs.tf @@ -0,0 +1,15 @@ +output "token" { + value = stackit_modelserving_token.this.token + sensitive = true + description = "The STACKIT Model Serving API token. Shown only on creation." +} + +output "token_id" { + value = stackit_modelserving_token.this.token_id + description = "ID of the Model Serving token." +} + +output "valid_until" { + value = stackit_modelserving_token.this.valid_until + description = "Expiry timestamp of the Model Serving token." +} diff --git a/modules/stackit/model-serving/buildingblock/provider.tf b/modules/stackit/model-serving/buildingblock/provider.tf new file mode 100644 index 00000000..485b4f82 --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/provider.tf @@ -0,0 +1,4 @@ +provider "stackit" { + service_account_email = var.service_account_email + use_oidc = true +} diff --git a/modules/stackit/model-serving/buildingblock/variables.tf b/modules/stackit/model-serving/buildingblock/variables.tf new file mode 100644 index 00000000..62fd2b4e --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/variables.tf @@ -0,0 +1,32 @@ +variable "service_account_email" { + type = string + description = "Email of the STACKIT service account used to issue the Model Serving token." +} + +variable "project_id" { + type = string + description = "STACKIT project ID the Model Serving token is issued in." +} + +variable "token_name" { + type = string + description = "Display name of the Model Serving token." +} + +variable "token_description" { + type = string + default = "Managed by meshStack." + description = "Description shown on the Model Serving token." +} + +variable "ttl_duration" { + type = string + default = "90d" + description = "Lifetime of the Model Serving token, e.g. '90d'." +} + +variable "region" { + type = string + default = null + description = "STACKIT region for the Model Serving token. Defaults to the provider region." +} diff --git a/modules/stackit/model-serving/buildingblock/versions.tf b/modules/stackit/model-serving/buildingblock/versions.tf new file mode 100644 index 00000000..249d5995 --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.88.0" + } + } +} diff --git a/reference-architectures/stackit-ai/README.md b/reference-architectures/stackit-ai/README.md new file mode 100644 index 00000000..992a2608 --- /dev/null +++ b/reference-architectures/stackit-ai/README.md @@ -0,0 +1,130 @@ +--- +name: STACKIT AI +description: > + A governed, observable AI platform on sovereign infrastructure: OpenWebUI as the user-facing + interface, LiteLLM as the model gateway enforcing per-tenant virtual keys and budgets, Langfuse + for tracing and evaluation, and STACKIT AI Model Serving as the sovereign model backend β€” ordered + per team as a meshStack building block. +cloudProviders: + - stackit +buildingBlocks: + - path: stackit/model-serving + role: Issues the per-tenant STACKIT AI Model Serving token and the matching LiteLLM virtual key. +--- + +# STACKIT AI + +## Overview + + + +Enterprise AI adoption stalls on three questions that a raw model endpoint does not answer: *who may +call which model*, *what did it cost*, and *what exactly was sent and returned*. This reference +architecture puts a governed gateway and an observability layer in front of sovereign model serving, +and makes tenant access a self-service building block instead of a shared API key passed around. + +**Target audience:** + +- **Platform engineers** who want to offer LLM access as a governed product β€” per-team keys, budgets + and model allow-lists β€” rather than handing out one shared credential. +- **Application teams** who need a stable, OpenAI-compatible endpoint and a chat UI without + operating model infrastructure. + +## Architecture Diagram + +The **AI platform** runs on SKE inside STACKIT. **OpenWebUI** is what users see; every call goes +through **LiteLLM**, the single choke point where virtual keys, budgets and model allow-lists are +enforced, and which routes inference to **STACKIT AI Model Serving**. **Langfuse** traces every call +for evaluation and usage attribution. Self-hosted **vLLM** is shown muted β€” it is an optional +backend, not required when the managed sovereign API is used. On the right, **meshStack** turns model +access into a catalog item: ordering the building block issues the tenant's key and ties usage back +to a project that carries budget and cost tags. + +![STACKIT AI reference architecture](stackit-ai.svg) + +## Governance and Observability + + + +| Concern | Where it is enforced | +|---------|----------------------| +| Which models a team may call | LiteLLM model allow-list, bound to the virtual key | +| Spend per team | LiteLLM budget on the virtual key; project budget in meshStack | +| Rate limiting / noisy neighbours | LiteLLM per-key rate limits | +| Prompt and response audit | Langfuse traces | +| Quality regression tracking | Langfuse evaluations | +| Cost attribution and chargeback | Langfuse usage β†’ meshStack project cost tags | +| Credential rotation | Building block re-order / token TTL | + +## Open Question: STACKIT-Specific or Pluggable Models? + + + +The demo stack was deliberately built so the infrastructure and model-serving layers are +**replaceable** β€” it ran on Scaleway, and STACKIT AI Model Serving can drop into the model layer. +That raises a scoping question for this reference architecture: + +**Option A β€” STACKIT AI (as drawn above).** One sovereign backend, the simplest story, fits the +`stackit/*` module namespace and the existing STACKIT reference architectures. + +**Option B β€” AI Platform with pluggable models (bring your own model).** LiteLLM already is the +abstraction layer, so the same architecture generalises: sovereign backends (STACKIT AI Model +Serving, self-hosted vLLM on SKE) alongside external ones (Azure OpenAI, any OpenAI-compatible API), +with routing policy deciding which tenant may reach outside the sovereign boundary. + +![Pluggable model backends variant](stackit-ai-pluggable.svg) + +Option B is the stronger platform story and makes the sovereignty boundary explicit rather than +implicit, but it widens the scope beyond a STACKIT reference architecture and needs a home outside +`modules/stackit/`. + +## How It Works + + + +1. The platform team deploys the AI platform components (OpenWebUI, LiteLLM, Langfuse) onto SKE. +2. LiteLLM is configured with STACKIT AI Model Serving as a backend and Langfuse as its trace sink. +3. The platform team registers the model-access building block against the AI-enabled landing zone. +4. An application team orders it in their project; the block issues a STACKIT Model Serving token and + a LiteLLM virtual key carrying that team's budget and model allow-list. +5. The team's users chat through OpenWebUI; every call is authorised at the gateway, traced in + Langfuse, and attributed back to the meshStack project. + +## Getting Started + +### Prerequisites + +| Requirement | Description | +|----------------------|-----------------------------------------------------------------------------| +| STACKIT organization | With AI Model Serving enabled and a service account permitted to issue tokens. | +| SKE cluster | A running STACKIT Kubernetes Engine cluster to host the platform components. | +| meshStack instance | With Terraform/OpenTofu IaC runtime configured. | + +### Deployment Order + + + +## Shared Responsibilities + +| Responsibility | Platform Team | Application Team | +|-------------------------------------------------------------|:---:|:---:| +| Operate the SKE cluster and the AI platform components | βœ… | ❌ | +| Decide which models are offered and to whom | βœ… | ❌ | +| Configure gateway budgets, rate limits and allow-lists | βœ… | ❌ | +| Register and maintain building block definitions | βœ… | ❌ | +| Order model access from the self-service catalog | ❌ | βœ… | +| Stay within the granted budget and model allow-list | ❌ | βœ… | +| Review own traces and evaluations in Langfuse | ❌ | βœ… | +| Build and operate the AI application or assistant | ❌ | βœ… | + +## Notes for the Session + + + +- Demo components in scope here: OpenWebUI, LiteLLM, Langfuse, STACKIT AI Model Serving. +- Dropped from the original demo for focus: FlowiseAI (agent/workflow builder), RAGFlow (RAG and data + layer). Worth deciding whether these return as a follow-up reference architecture. +- Original demo ran on Scaleway; only the model-serving and infrastructure layers need swapping. +- Hub modules still missing for the platform components themselves β€” only `stackit/model-serving` is + scaffolded so far. diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot b/reference-architectures/stackit-ai/stackit-ai-pluggable.dot new file mode 100644 index 00000000..d416e055 --- /dev/null +++ b/reference-architectures/stackit-ai/stackit-ai-pluggable.dot @@ -0,0 +1,62 @@ +/* + * Variant under discussion: the same gateway with pluggable model backends (bring your own model). + * Conventions: .agents/references/diagrams.md β€” render with: task diagrams + */ +digraph stackit_ai_pluggable { + rankdir=TB + compound=true + splines=ortho + forcelabels=true + bgcolor="white" + nodesep=0.55 + ranksep=0.85 + + node [shape=box style="rounded,filled" fontname="Helvetica" fontsize=11 + fillcolor="#ffffff" color="#a2abb8" penwidth=1.1 margin="0.20,0.11"] + edge [fontname="Helvetica" fontsize=9 fontcolor="#697180" color="#8b95a3" arrowsize=0.7] + + USER [label="πŸ‘€ Application team" fillcolor="#ffffff" color="#a2abb8"] + WEBUI [label="πŸ’¬ OpenWebUI\nuser-facing chat interface" fillcolor="#e5f2ea" color="#85bfa0"] + LITELLM [label="πŸšͺ LiteLLM\none API Β· virtual keys Β· budgets Β· allow-list" fillcolor="#e5f2ea" color="#85bfa0"] + LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation Β· usage" fillcolor="#e5f2ea" color="#85bfa0"] + BBD_M [label="πŸ“¦ model-access BBD\nself-service, per tenant" fillcolor="#ecedfb" color="#9aa2e6"] + + { rank=same; LITELLM -> LANGFUSE [style=invis] } + + USER -> WEBUI [xlabel="chats"] + WEBUI -> LITELLM [xlabel="completions"] + LITELLM -> LANGFUSE [xlabel="traces every call" style=dashed constraint=false] + BBD_M -> LITELLM [xlabel="issues virtual key" style=dotted constraint=false] + + subgraph cluster_sovereign { + label=" Sovereign backends" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + MODEL [label="🧠 STACKIT AI Model Serving\nmanaged sovereign LLM API" fillcolor="#eef2f6" color="#93a7bb"] + VLLM [label="🧠 vLLM on SKE\nself-hosted models" fillcolor="#eef2f6" color="#93a7bb"] + + { rank=same; MODEL -> VLLM [style=invis] } + } + + subgraph cluster_byom { + label=" Bring your own model Β· outside the sovereign boundary" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + AZURE [label="🧠 Azure OpenAI" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + OTHER [label="🧠 Any OpenAI-compatible API" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + + { rank=same; AZURE -> OTHER [style=invis] } + } + + LITELLM -> MODEL [xlabel="routes by policy"] + LITELLM -> VLLM + LITELLM -> AZURE [xlabel="or any external model" lhead=cluster_byom style=dashed] +} diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg b/reference-architectures/stackit-ai/stackit-ai-pluggable.svg new file mode 100644 index 00000000..50fe7af9 --- /dev/null +++ b/reference-architectures/stackit-ai/stackit-ai-pluggable.svg @@ -0,0 +1,131 @@ + + + + +stackit_ai_pluggable + + +cluster_sovereign + +  Sovereign backends + + +cluster_byom + +  Bring your own model Β· outside the sovereign boundary + + + +USER + +πŸ‘€ Application team + + + +WEBUI + +πŸ’¬ OpenWebUI +user-facing chat interface + + + +USER->WEBUI + + +chats + + + +LITELLM + +πŸšͺ LiteLLM +one API Β· virtual keys Β· budgets Β· allow-list + + + +WEBUI->LITELLM + + +completions + + + +LANGFUSE + +πŸ“ˆ Langfuse +tracing Β· evaluation Β· usage + + + + +LITELLM->LANGFUSE + + +traces every call + + + +MODEL + +🧠 STACKIT AI Model Serving +managed sovereign LLM API + + + +LITELLM->MODEL + + +routes by policy + + + +VLLM + +🧠 vLLM on SKE +self-hosted models + + + +LITELLM->VLLM + + + + + +AZURE + +🧠 Azure OpenAI + + + +LITELLM->AZURE + + +or any external model + + + +BBD_M + +πŸ“¦ model-access BBD +self-service, per tenant + + + +BBD_M->LITELLM + + +issues virtual key + + + + +OTHER + +🧠 Any OpenAI-compatible API + + + + diff --git a/reference-architectures/stackit-ai/stackit-ai.dot b/reference-architectures/stackit-ai/stackit-ai.dot new file mode 100644 index 00000000..6039d5cf --- /dev/null +++ b/reference-architectures/stackit-ai/stackit-ai.dot @@ -0,0 +1,74 @@ +/* + * STACKIT AI reference architecture β€” governed, observable sovereign model serving. + * Conventions: .agents/references/diagrams.md β€” render with: task diagrams + */ +digraph stackit_ai { + rankdir=TB + splines=ortho + forcelabels=true + bgcolor="white" + nodesep=0.55 + ranksep=0.85 + + node [shape=box style="rounded,filled" fontname="Helvetica" fontsize=11 + fillcolor="#ffffff" color="#a2abb8" penwidth=1.1 margin="0.20,0.11"] + edge [fontname="Helvetica" fontsize=9 fontcolor="#697180" color="#8b95a3" arrowsize=0.7] + + USER [label="πŸ‘€ Application team" fillcolor="#ffffff" color="#a2abb8"] + + subgraph cluster_stackit { + label=" STACKIT β€” sovereign infrastructure" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + subgraph cluster_stack { + label=" AI Platform Β· deployed on SKE" + labeljust=l + fontname="Helvetica" fontsize=11 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=14 + + WEBUI [label="πŸ’¬ OpenWebUI\nuser-facing chat interface" fillcolor="#e5f2ea" color="#85bfa0"] + LITELLM [label="πŸšͺ LiteLLM\nvirtual keys Β· budgets Β· model allow-list" fillcolor="#e5f2ea" color="#85bfa0"] + LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation Β· usage" fillcolor="#e5f2ea" color="#85bfa0"] + VLLM [label="🧠 vLLM\nself-hosted models Β· optional" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + + { rank=same; LITELLM -> LANGFUSE [style=invis] } + + WEBUI -> LITELLM [xlabel="completions"] + LITELLM -> LANGFUSE [xlabel="traces every call" style=dashed constraint=false] + LITELLM -> VLLM [xlabel="optional backend" style=dashed] + } + + MODEL [label="🧠 STACKIT AI Model Serving\nmanaged sovereign LLM API" fillcolor="#eef2f6" color="#93a7bb"] + + LITELLM -> MODEL [xlabel="routes inference to"] + } + + subgraph cluster_mesh { + label=" meshStack β€” self-service and governance" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + PLAT [label="πŸ›°οΈ SKE Platform" fillcolor="#ecedfb" color="#9aa2e6"] + LZ [label="πŸ›¬ Landing Zone\nAI-enabled" fillcolor="#ecedfb" color="#9aa2e6"] + BBD_M [label="πŸ“¦ stackit/model-serving BBD\nself-service model access" fillcolor="#ecedfb" color="#9aa2e6"] + PROJ [label="πŸ—‚οΈ Project Γ—N\ncarries budget and cost tags" fillcolor="#e5f2ea" color="#85bfa0"] + + { rank=same; LZ -> BBD_M [style=invis] } + + PLAT -> LZ + LZ -> PROJ [xlabel="applies to"] + BBD_M -> PROJ [xlabel="ordered in" constraint=false] + } + + USER -> WEBUI [xlabel="chats"] + BBD_M -> LITELLM [xlabel="issues virtual key" style=dotted constraint=false] + BBD_M -> MODEL [xlabel="issues serving token" style=dotted constraint=false] + LANGFUSE -> PROJ [xlabel="usage attribution" style=dotted constraint=false] +} diff --git a/reference-architectures/stackit-ai/stackit-ai.svg b/reference-architectures/stackit-ai/stackit-ai.svg new file mode 100644 index 00000000..e67258b0 --- /dev/null +++ b/reference-architectures/stackit-ai/stackit-ai.svg @@ -0,0 +1,171 @@ + + + + +stackit_ai + + +cluster_stackit + +  STACKIT β€” sovereign infrastructure + + +cluster_stack + +  AI Platform Β· deployed on SKE + + +cluster_mesh + +  meshStack β€” self-service and governance + + + +USER + +πŸ‘€ Application team + + + +WEBUI + +πŸ’¬ OpenWebUI +user-facing chat interface + + + +USER->WEBUI + + +chats + + + +LITELLM + +πŸšͺ LiteLLM +virtual keys Β· budgets Β· model allow-list + + + +WEBUI->LITELLM + + +completions + + + +LANGFUSE + +πŸ“ˆ Langfuse +tracing Β· evaluation Β· usage + + + + +LITELLM->LANGFUSE + + +traces every call + + + +VLLM + +🧠 vLLM +self-hosted models Β· optional + + + +LITELLM->VLLM + + +optional backend + + + +MODEL + +🧠 STACKIT AI Model Serving +managed sovereign LLM API + + + +LITELLM->MODEL + + +routes inference to + + + +PROJ + +πŸ—‚οΈ Project Γ—N +carries budget and cost tags + + + +LANGFUSE->PROJ + + +usage attribution + + + +PLAT + +πŸ›°οΈ SKE Platform + + + +LZ + +πŸ›¬ Landing Zone +AI-enabled + + + +PLAT->LZ + + + + + +BBD_M + +πŸ“¦ stackit/model-serving BBD +self-service model access + + + + +LZ->PROJ + + +applies to + + + +BBD_M->LITELLM + + +issues virtual key + + + +BBD_M->MODEL + + +issues serving token + + + +BBD_M->PROJ + + +ordered in + + + From 4bd8787b70a5d3023dd46bd3df860e89c36404db Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 09:27:48 +0200 Subject: [PATCH 02/60] feat(ref-arch): add pluggability seams and one-click delivery model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the two questions raised in review: where the platform components run, and how to let users choose STACKIT or Azure for both runtime and models. Two orthogonal seams, so this stays one reference architecture instead of forking per cloud: - Runtime: components are Helm building blocks declaring supportedPlatforms kubernetes, installed into whichever landing zone hands out namespaces (SKE, AKS, any conformant cluster). Precedent: kubernetes/manifest. - Model: one small model-access module per provider, all exposing the same OpenAI-compatible endpoint plus credential shape to LiteLLM. Adds the one-click delivery model: an ai-platform block installs the stack once, then LiteLLM is registered as a meshStack platform so model access becomes a landing-zone-governed self-service order. This is expressible today via spec.config.custom.platform_type_ref β€” modules/stackit already registers STACKIT itself as a custom platform type with stackit/project doing the provisioning. New diagrams: stackit-ai-oneclick (lifecycle split) and a rewritten stackit-ai-pluggable showing both seams as parallel swap-slots. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/stackit-ai/README.md | 163 +++++++----- .../stackit-ai/stackit-ai-oneclick.dot | 75 ++++++ .../stackit-ai/stackit-ai-oneclick.svg | 166 ++++++++++++ .../stackit-ai/stackit-ai-pluggable.dot | 80 ++++-- .../stackit-ai/stackit-ai-pluggable.svg | 246 +++++++++++------- 5 files changed, 542 insertions(+), 188 deletions(-) create mode 100644 reference-architectures/stackit-ai/stackit-ai-oneclick.dot create mode 100644 reference-architectures/stackit-ai/stackit-ai-oneclick.svg diff --git a/reference-architectures/stackit-ai/README.md b/reference-architectures/stackit-ai/README.md index 992a2608..1de5c461 100644 --- a/reference-architectures/stackit-ai/README.md +++ b/reference-architectures/stackit-ai/README.md @@ -1,15 +1,14 @@ --- name: STACKIT AI description: > - A governed, observable AI platform on sovereign infrastructure: OpenWebUI as the user-facing - interface, LiteLLM as the model gateway enforcing per-tenant virtual keys and budgets, Langfuse - for tracing and evaluation, and STACKIT AI Model Serving as the sovereign model backend β€” ordered - per team as a meshStack building block. + An opinionated, one-click AI platform: OpenWebUI, LiteLLM and Langfuse installed into a Kubernetes + namespace landing zone, with LiteLLM registered as a meshStack platform so application teams order + governed model access β€” budget, allowed models and tracing included β€” as a self-service item. cloudProviders: - stackit buildingBlocks: - path: stackit/model-serving - role: Issues the per-tenant STACKIT AI Model Serving token and the matching LiteLLM virtual key. + role: Issues the STACKIT AI Model Serving credential that LiteLLM routes sovereign inference through. --- # STACKIT AI @@ -18,30 +17,84 @@ buildingBlocks: -Enterprise AI adoption stalls on three questions that a raw model endpoint does not answer: *who may -call which model*, *what did it cost*, and *what exactly was sent and returned*. This reference -architecture puts a governed gateway and an observability layer in front of sovereign model serving, -and makes tenant access a self-service building block instead of a shared API key passed around. +Enterprise AI adoption stalls on three questions a raw model endpoint does not answer: *who may call +which model*, *what did it cost*, and *what exactly was sent and returned*. This reference +architecture answers them with opinionated defaults rather than a toolkit: one order installs the +gateway, chat UI and observability stack, and a second turns model access into a governed, +self-service catalog item. **Target audience:** - **Platform engineers** who want to offer LLM access as a governed product β€” per-team keys, budgets - and model allow-lists β€” rather than handing out one shared credential. -- **Application teams** who need a stable, OpenAI-compatible endpoint and a chat UI without - operating model infrastructure. + and model allow-lists β€” instead of handing out one shared credential. +- **Application teams** who need a stable OpenAI-compatible endpoint and a chat UI without operating + model infrastructure. -## Architecture Diagram +## Architecture The **AI platform** runs on SKE inside STACKIT. **OpenWebUI** is what users see; every call goes through **LiteLLM**, the single choke point where virtual keys, budgets and model allow-lists are enforced, and which routes inference to **STACKIT AI Model Serving**. **Langfuse** traces every call -for evaluation and usage attribution. Self-hosted **vLLM** is shown muted β€” it is an optional -backend, not required when the managed sovereign API is used. On the right, **meshStack** turns model -access into a catalog item: ordering the building block issues the tenant's key and ties usage back -to a project that carries budget and cost tags. +for evaluation and usage attribution. Self-hosted **vLLM** is shown muted β€” an optional backend, not +required when the managed sovereign API is used. ![STACKIT AI reference architecture](stackit-ai.svg) +## Delivery Model: Two Orders + +The architecture is deliberately opinionated so it is ready to go rather than assembled. It splits +into one platform-team order and one application-team order. + +![One-click delivery model](stackit-ai-oneclick.svg) + +**β‘  Platform team, once.** An `ai-platform` building block deploys OpenWebUI, LiteLLM and Langfuse by +Helm into a tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by +convention: LiteLLM points at Langfuse for tracing, OpenWebUI points at LiteLLM, model backends are +registered from the credentials it was given. + +**β‘‘ Application team, per team.** LiteLLM is then registered as a **meshStack platform**, so ordering +model access is a normal self-service action: the landing zone carries the policy (allowed models, +budget tier), and the building block creates the LiteLLM team and virtual key behind it. + +### Why LiteLLM Works as a meshStack Platform + +LiteLLM's native concepts already form a tenancy model, which is what meshStack replicates into: + +| meshStack | LiteLLM | +|-----------|---------| +| Project | Team | +| Landing zone | Allowed models, budget and rate-limit tier | +| Tenant credential | Virtual key | +| Project roles | Team membership | + +This is expressible today: `meshstack_platform` supports `spec.config.custom.platform_type_ref`, and +there is a precedent in this repo β€” [`modules/stackit`](../../modules/stackit) registers STACKIT +itself as a **custom** platform type, with the actual tenant provisioning done by the +[`stackit/project`](../../modules/stackit/project) building block. A LiteLLM platform would follow +the same shape, with a `litellm/team` building block in place of `stackit/project`. + +## Pluggability: Two Independent Seams + +The demo stack was built so infrastructure and model serving are replaceable. That generalises into +**two orthogonal seams** β€” and because they are orthogonal, this should stay *one* reference +architecture rather than forking into a STACKIT and an Azure variant. Running on SKE while calling +Azure OpenAI, or on AKS while calling STACKIT, are both valid combinations. + +| Seam | Contract | Chosen by | Implementations | +|------|----------|-----------|-----------------| +| **Runtime** β€” where the components run | A landing zone that hands out Kubernetes namespaces; the blocks declare `supportedPlatforms: kubernetes` and never name a cloud | The landing zone the platform team orders into | STACKIT SKE, Azure AKS, any conformant cluster | +| **Model** β€” where inference happens | An OpenAI-compatible endpoint plus credential, surfaced as a LiteLLM `model_list` entry | LiteLLM routing policy, fed by one model-access block per provider | `stackit/model-serving`, an Azure OpenAI equivalent, self-hosted vLLM | + +![Pluggable seams variant](stackit-ai-pluggable.svg) + +Adding a cloud therefore means adding one small model-access module with the same output shape β€” not +changing the architecture. The runtime seam needs no per-cloud work at all: the precedent is +[`kubernetes/manifest`](../../modules/kubernetes/manifest), a runtime-agnostic Helm building block +that takes a kubeconfig and declares `supportedPlatforms: kubernetes`. + +This architecture **consumes** a Kubernetes cluster, it does not provision one β€” which is why +[`stackit-kubernetes`](../stackit-kubernetes) stays a separate, companion reference architecture. + ## Governance and Observability @@ -56,50 +109,34 @@ to a project that carries budget and cost tags. | Cost attribution and chargeback | Langfuse usage β†’ meshStack project cost tags | | Credential rotation | Building block re-order / token TTL | -## Open Question: STACKIT-Specific or Pluggable Models? - - - -The demo stack was deliberately built so the infrastructure and model-serving layers are -**replaceable** β€” it ran on Scaleway, and STACKIT AI Model Serving can drop into the model layer. -That raises a scoping question for this reference architecture: - -**Option A β€” STACKIT AI (as drawn above).** One sovereign backend, the simplest story, fits the -`stackit/*` module namespace and the existing STACKIT reference architectures. - -**Option B β€” AI Platform with pluggable models (bring your own model).** LiteLLM already is the -abstraction layer, so the same architecture generalises: sovereign backends (STACKIT AI Model -Serving, self-hosted vLLM on SKE) alongside external ones (Azure OpenAI, any OpenAI-compatible API), -with routing policy deciding which tenant may reach outside the sovereign boundary. - -![Pluggable model backends variant](stackit-ai-pluggable.svg) - -Option B is the stronger platform story and makes the sovereignty boundary explicit rather than -implicit, but it widens the scope beyond a STACKIT reference architecture and needs a home outside -`modules/stackit/`. - -## How It Works - - - -1. The platform team deploys the AI platform components (OpenWebUI, LiteLLM, Langfuse) onto SKE. -2. LiteLLM is configured with STACKIT AI Model Serving as a backend and Langfuse as its trace sink. -3. The platform team registers the model-access building block against the AI-enabled landing zone. -4. An application team orders it in their project; the block issues a STACKIT Model Serving token and - a LiteLLM virtual key carrying that team's budget and model allow-list. -5. The team's users chat through OpenWebUI; every call is authorised at the gateway, traced in - Langfuse, and attributed back to the meshStack project. +## Open Questions + + + +1. **Scope and name.** If both seams are real, this is an *AI platform* reference architecture with + `cloudProviders: [stackit, azure]`, not a STACKIT-only one β€” STACKIT would be the sovereign + reference instantiation. That implies renaming the folder and finding a home for the + runtime-agnostic modules outside `modules/stackit/`. +2. **Bootstrap ordering.** The LiteLLM platform can only be registered once LiteLLM is reachable (URL + plus admin credential). `stackit-landingzone` already registers a platform and orders a building + block instance in one apply, so there is a precedent β€” but the dependency needs designing. +3. **Provisioning mechanism.** There is no LiteLLM Terraform provider as far as we know, so the + `litellm/team` block would drive LiteLLM's admin API. Needs confirming. +4. **Metering.** The custom platform type accepts metering configuration; whether LiteLLM spend can + feed meshStack chargeback is unresolved. +5. **Tenancy of the UI.** One shared OpenWebUI with per-user keys, or one instance per team? +6. **Dropped from the demo.** FlowiseAI (agent/workflow builder) and RAGFlow (RAG and data layer) were + left out to keep the focus on serving, observability and governance. Follow-up architecture? ## Getting Started ### Prerequisites -| Requirement | Description | -|----------------------|-----------------------------------------------------------------------------| -| STACKIT organization | With AI Model Serving enabled and a service account permitted to issue tokens. | -| SKE cluster | A running STACKIT Kubernetes Engine cluster to host the platform components. | -| meshStack instance | With Terraform/OpenTofu IaC runtime configured. | +| Requirement | Description | +|----------------------|--------------------------------------------------------------------------------| +| Kubernetes landing zone | A meshStack landing zone providing namespaces β€” STACKIT SKE or Azure AKS. | +| Model backend | STACKIT AI Model Serving enabled, or an equivalent OpenAI-compatible endpoint. | +| meshStack instance | With Terraform/OpenTofu IaC runtime configured. | ### Deployment Order @@ -109,9 +146,10 @@ implicit, but it widens the scope beyond a STACKIT reference architecture and ne | Responsibility | Platform Team | Application Team | |-------------------------------------------------------------|:---:|:---:| -| Operate the SKE cluster and the AI platform components | βœ… | ❌ | +| Operate the Kubernetes cluster hosting the platform | βœ… | ❌ | +| Install and upgrade OpenWebUI, LiteLLM and Langfuse | βœ… | ❌ | | Decide which models are offered and to whom | βœ… | ❌ | -| Configure gateway budgets, rate limits and allow-lists | βœ… | ❌ | +| Define landing zones: allowed models, budgets, rate limits | βœ… | ❌ | | Register and maintain building block definitions | βœ… | ❌ | | Order model access from the self-service catalog | ❌ | βœ… | | Stay within the granted budget and model allow-list | ❌ | βœ… | @@ -122,9 +160,8 @@ implicit, but it widens the scope beyond a STACKIT reference architecture and ne -- Demo components in scope here: OpenWebUI, LiteLLM, Langfuse, STACKIT AI Model Serving. -- Dropped from the original demo for focus: FlowiseAI (agent/workflow builder), RAGFlow (RAG and data - layer). Worth deciding whether these return as a follow-up reference architecture. -- Original demo ran on Scaleway; only the model-serving and infrastructure layers need swapping. +- Demo components in scope: OpenWebUI, LiteLLM, Langfuse, STACKIT AI Model Serving. +- Original demo ran on Scaleway; only the runtime and model-serving layers need swapping. - Hub modules still missing for the platform components themselves β€” only `stackit/model-serving` is - scaffolded so far. + scaffolded, as a minimal first cut around `stackit_modelserving_token` (the only AI-specific + resource in the STACKIT provider, v0.88.0). diff --git a/reference-architectures/stackit-ai/stackit-ai-oneclick.dot b/reference-architectures/stackit-ai/stackit-ai-oneclick.dot new file mode 100644 index 00000000..b833b38f --- /dev/null +++ b/reference-architectures/stackit-ai/stackit-ai-oneclick.dot @@ -0,0 +1,75 @@ +/* + * Delivery model under discussion: opinionated one-click install, then LiteLLM registered as a + * custom meshStack platform so model access itself becomes a landing-zone-governed order. + * Conventions: .agents/references/diagrams.md β€” render with: task diagrams + */ +digraph stackit_ai_oneclick { + rankdir=TB + splines=ortho + forcelabels=true + bgcolor="white" + nodesep=0.55 + ranksep=0.85 + + node [shape=box style="rounded,filled" fontname="Helvetica" fontsize=11 + fillcolor="#ffffff" color="#a2abb8" penwidth=1.1 margin="0.20,0.11"] + edge [fontname="Helvetica" fontsize=9 fontcolor="#697180" color="#8b95a3" arrowsize=0.7] + + subgraph cluster_once { + label=" β‘  Platform team Β· orders once" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + BBD_STACK [label="πŸ“¦ ai-platform BBD\nopinionated Helm wiring" fillcolor="#ecedfb" color="#9aa2e6"] + NS [label="☸️ Tenant namespace\nfrom the Kubernetes landing zone" fillcolor="#e5f2ea" color="#85bfa0"] + STACK [label="πŸšͺ OpenWebUI Β· LiteLLM Β· Langfuse\nwired together by convention" fillcolor="#e5f2ea" color="#85bfa0"] + PLATFORM [label="πŸ›°οΈ LiteLLM as meshStack platform\ncustom platform type" fillcolor="#ecedfb" color="#9aa2e6"] + LZAI [label="πŸ›¬ AI landing zones\nallowed models Β· budget tier" fillcolor="#ecedfb" color="#9aa2e6"] + + { rank=same; PLATFORM -> LZAI [style=invis] } + + BBD_STACK -> NS [xlabel="deploys into"] + NS -> STACK [xlabel="hosts"] + STACK -> PLATFORM [xlabel="registered as" style=dotted] + PLATFORM -> LZAI + } + + subgraph cluster_team { + label=" β‘‘ Application team Β· one click, per team" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + BBD_TEAM [label="πŸ“¦ litellm/team BBD" fillcolor="#ecedfb" color="#9aa2e6"] + KEY [label="πŸ”‘ LiteLLM team and virtual key Γ—N\nmodels and budget from the landing zone" fillcolor="#e5f2ea" color="#85bfa0"] + USER [label="πŸ‘€ Application team\nchats in OpenWebUI" fillcolor="#ffffff" color="#a2abb8"] + + { rank=same; KEY -> USER [style=invis] } + + BBD_TEAM -> KEY [xlabel="creates"] + KEY -> USER [xlabel="grants access" style=dashed] + } + + subgraph cluster_existing { + label=" β‘’ Existing infrastructure" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + K8SLZ [label="πŸ›¬ Kubernetes landing zone\nSTACKIT SKE or Azure AKS" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + MODELS [label="🧠 Model backends\nSTACKIT Β· Azure Β· self-hosted" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + + { rank=same; K8SLZ -> MODELS [style=invis] } + } + + K8SLZ -> NS [xlabel="provides" style=dotted constraint=false] + LZAI -> BBD_TEAM [xlabel="governs" style=dotted] + KEY -> STACK [xlabel="authorises calls" style=dashed constraint=false] + STACK -> MODELS [xlabel="routes by policy" constraint=false] +} diff --git a/reference-architectures/stackit-ai/stackit-ai-oneclick.svg b/reference-architectures/stackit-ai/stackit-ai-oneclick.svg new file mode 100644 index 00000000..ae887a4f --- /dev/null +++ b/reference-architectures/stackit-ai/stackit-ai-oneclick.svg @@ -0,0 +1,166 @@ + + + + +stackit_ai_oneclick + + +cluster_once + +  β‘  Platform team Β· orders once + + +cluster_team + +  β‘‘ Application team Β· one click, per team + + +cluster_existing + +  β‘’ Existing infrastructure + + + +BBD_STACK + +πŸ“¦ ai-platform BBD +opinionated Helm wiring + + + +NS + +☸️ Tenant namespace +from the Kubernetes landing zone + + + +BBD_STACK->NS + + +deploys into + + + +STACK + +πŸšͺ OpenWebUI Β· LiteLLM Β· Langfuse +wired together by convention + + + +NS->STACK + + +hosts + + + +PLATFORM + +πŸ›°οΈ LiteLLM as meshStack platform +custom platform type + + + +STACK->PLATFORM + + +registered as + + + +MODELS + +🧠 Model backends +STACKIT Β· Azure Β· self-hosted + + + +STACK->MODELS + + +routes by policy + + + +LZAI + +πŸ›¬ AI landing zones +allowed models Β· budget tier + + + + +PLATFORM->LZAI + + + + + +BBD_TEAM + +πŸ“¦ litellm/team BBD + + + +LZAI->BBD_TEAM + + +governs + + + +KEY + +πŸ”‘ LiteLLM team and virtual key Γ—N +models and budget from the landing zone + + + +BBD_TEAM->KEY + + +creates + + + +KEY->STACK + + +authorises calls + + + +USER + +πŸ‘€ Application team +chats in OpenWebUI + + + + +KEY->USER + + +grants access + + + +K8SLZ + +πŸ›¬ Kubernetes landing zone +STACKIT SKE or Azure AKS + + + +K8SLZ->NS + + +provides + + + + diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot b/reference-architectures/stackit-ai/stackit-ai-pluggable.dot index d416e055..908a0df9 100644 --- a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot +++ b/reference-architectures/stackit-ai/stackit-ai-pluggable.dot @@ -1,5 +1,6 @@ /* - * Variant under discussion: the same gateway with pluggable model backends (bring your own model). + * Variant under discussion: two pluggable seams β€” the Kubernetes platform whose landing zone + * provides the namespace, and the model backends LiteLLM routes to. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ digraph stackit_ai_pluggable { @@ -15,48 +16,75 @@ digraph stackit_ai_pluggable { fillcolor="#ffffff" color="#a2abb8" penwidth=1.1 margin="0.20,0.11"] edge [fontname="Helvetica" fontsize=9 fontcolor="#697180" color="#8b95a3" arrowsize=0.7] - USER [label="πŸ‘€ Application team" fillcolor="#ffffff" color="#a2abb8"] - WEBUI [label="πŸ’¬ OpenWebUI\nuser-facing chat interface" fillcolor="#e5f2ea" color="#85bfa0"] - LITELLM [label="πŸšͺ LiteLLM\none API Β· virtual keys Β· budgets Β· allow-list" fillcolor="#e5f2ea" color="#85bfa0"] - LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation Β· usage" fillcolor="#e5f2ea" color="#85bfa0"] - BBD_M [label="πŸ“¦ model-access BBD\nself-service, per tenant" fillcolor="#ecedfb" color="#9aa2e6"] + USER [label="πŸ‘€ Application team" fillcolor="#ffffff" color="#a2abb8"] - { rank=same; LITELLM -> LANGFUSE [style=invis] } + subgraph cluster_mesh { + label=" meshStack" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 - USER -> WEBUI [xlabel="chats"] - WEBUI -> LITELLM [xlabel="completions"] - LITELLM -> LANGFUSE [xlabel="traces every call" style=dashed constraint=false] - BBD_M -> LITELLM [xlabel="issues virtual key" style=dotted constraint=false] + BBD_COMP [label="πŸ“¦ openwebui Β· litellm Β· langfuse BBDs\nsupportedPlatforms: kubernetes" fillcolor="#ecedfb" color="#9aa2e6"] + LZ [label="πŸ›¬ Landing Zone\nprovides the namespace" fillcolor="#ecedfb" color="#9aa2e6"] + BBD_MODEL [label="πŸ“¦ model-access BBD\none per provider, same output shape" fillcolor="#ecedfb" color="#9aa2e6"] - subgraph cluster_sovereign { - label=" Sovereign backends" + { rank=same; BBD_COMP -> LZ -> BBD_MODEL [style=invis] } + } + + subgraph cluster_platform { + label=" AI Platform Β· Helm into a tenant namespace" labeljust=l fontname="Helvetica" fontsize=12 fontcolor="#697180" style="rounded" color="#d5dae0" margin=18 - MODEL [label="🧠 STACKIT AI Model Serving\nmanaged sovereign LLM API" fillcolor="#eef2f6" color="#93a7bb"] - VLLM [label="🧠 vLLM on SKE\nself-hosted models" fillcolor="#eef2f6" color="#93a7bb"] + WEBUI [label="πŸ’¬ OpenWebUI" fillcolor="#e5f2ea" color="#85bfa0"] + LITELLM [label="πŸšͺ LiteLLM\nvirtual keys Β· budgets Β· allow-list" fillcolor="#e5f2ea" color="#85bfa0"] + LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation" fillcolor="#e5f2ea" color="#85bfa0"] + + { rank=same; WEBUI -> LITELLM -> LANGFUSE [style=invis] } - { rank=same; MODEL -> VLLM [style=invis] } + WEBUI -> LITELLM [xlabel="completions"] + LITELLM -> LANGFUSE [xlabel="traces" style=dashed constraint=false] } - subgraph cluster_byom { - label=" Bring your own model Β· outside the sovereign boundary" + subgraph cluster_runtime { + label=" Runtime seam Β· any meshStack Kubernetes platform" labeljust=l fontname="Helvetica" fontsize=12 fontcolor="#697180" style="rounded" color="#d5dae0" margin=18 - AZURE [label="🧠 Azure OpenAI" - fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] - OTHER [label="🧠 Any OpenAI-compatible API" - fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + SKE [label="☸️ STACKIT SKE\nsovereign" fillcolor="#eef2f6" color="#93a7bb"] + AKS [label="☸️ Azure AKS" fillcolor="#eef2f6" color="#93a7bb"] + ANYK8S [label="☸️ Any conformant cluster" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] - { rank=same; AZURE -> OTHER [style=invis] } + { rank=same; SKE -> AKS -> ANYK8S [style=invis] } } - LITELLM -> MODEL [xlabel="routes by policy"] - LITELLM -> VLLM - LITELLM -> AZURE [xlabel="or any external model" lhead=cluster_byom style=dashed] + subgraph cluster_models { + label=" Model seam Β· OpenAI-compatible backends" + labeljust=l + fontname="Helvetica" fontsize=12 fontcolor="#697180" + style="rounded" color="#d5dae0" + margin=18 + + STACKITM [label="🧠 STACKIT AI Model Serving\nsovereign" fillcolor="#eef2f6" color="#93a7bb"] + AZUREM [label="🧠 Azure OpenAI" fillcolor="#eef2f6" color="#93a7bb"] + VLLM [label="🧠 Self-hosted vLLM" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + + { rank=same; STACKITM -> AZUREM -> VLLM [style=invis] } + } + + USER -> WEBUI [xlabel="chats"] + + BBD_COMP -> WEBUI [xlabel="deploys via Helm" style=dotted lhead=cluster_platform] + LZ -> SKE [xlabel="backed by" style=dotted lhead=cluster_runtime constraint=false] + BBD_MODEL -> LITELLM [xlabel="virtual key and model list" style=dotted constraint=false] + + LANGFUSE -> SKE [xlabel="namespace runs on" ltail=cluster_platform lhead=cluster_runtime] + LITELLM -> STACKITM [xlabel="routes by policy" lhead=cluster_models] } diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg b/reference-architectures/stackit-ai/stackit-ai-pluggable.svg index 50fe7af9..07dc8233 100644 --- a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg +++ b/reference-architectures/stackit-ai/stackit-ai-pluggable.svg @@ -1,131 +1,179 @@ - - + + stackit_ai_pluggable - - -cluster_sovereign - -  Sovereign backends - - -cluster_byom - -  Bring your own model Β· outside the sovereign boundary + + +cluster_mesh + +  meshStack + + +cluster_platform + +  AI Platform Β· Helm into a tenant namespace + + +cluster_runtime + +  Runtime seam Β· any meshStack Kubernetes platform + + +cluster_models + +  Model seam Β· OpenAI-compatible backends USER - -πŸ‘€ Application team + +πŸ‘€ Application team - + WEBUI - -πŸ’¬ OpenWebUI -user-facing chat interface + +πŸ’¬ OpenWebUI - + USER->WEBUI - - -chats + + +chats - + + +BBD_COMP + +πŸ“¦ openwebui Β· litellm Β· langfuse BBDs +supportedPlatforms: kubernetes + + +LZ + +πŸ›¬ Landing Zone +provides the namespace + + + + +BBD_COMP->WEBUI + + +deploys via Helm + + + +BBD_MODEL + +πŸ“¦ model-access BBD +one per provider, same output shape + + + + +SKE + +☸️ STACKIT SKE +sovereign + + + +LZ->SKE + + +backed by + + + LITELLM - -πŸšͺ LiteLLM -one API Β· virtual keys Β· budgets Β· allow-list + +πŸšͺ LiteLLM +virtual keys Β· budgets Β· allow-list + + +BBD_MODEL->LITELLM + + +virtual key and model list + + - + WEBUI->LITELLM - - -completions + + +completions - + LANGFUSE - -πŸ“ˆ Langfuse -tracing Β· evaluation Β· usage + +πŸ“ˆ Langfuse +tracing Β· evaluation - + LITELLM->LANGFUSE - - -traces every call - - - -MODEL - -🧠 STACKIT AI Model Serving -managed sovereign LLM API - - - -LITELLM->MODEL - - -routes by policy - + + +traces + + + +STACKITM + +🧠 STACKIT AI Model Serving +sovereign + + + +LITELLM->STACKITM + + +routes by policy + + + +LANGFUSE->SKE + + +namespace runs on + + + +AKS + +☸️ Azure AKS + + + + +ANYK8S + +☸️ Any conformant cluster + + + + +AZUREM + +🧠 Azure OpenAI + + - + VLLM - -🧠 vLLM on SKE -self-hosted models - - - -LITELLM->VLLM - - - - - -AZURE - -🧠 Azure OpenAI - - - -LITELLM->AZURE - - -or any external model - - - -BBD_M - -πŸ“¦ model-access BBD -self-service, per tenant - - - -BBD_M->LITELLM - - -issues virtual key - - - - -OTHER - -🧠 Any OpenAI-compatible API + +🧠 Self-hosted vLLM - + From 92591d496add051376c9a278bc565f072574a44b Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 09:39:43 +0200 Subject: [PATCH 03/60] refactor(ref-arch): drop the chat UI from the AI platform core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenWebUI is an application, not platform plumbing. The reference architecture delivers a governed API β€” LiteLLM for policy, Langfuse for observability β€” and what teams build on that endpoint is their own concern. Concretely it was removed because: - its own user/group model permissions are a second policy store competing with LiteLLM, and "who may call which model" must have exactly one home - its built-in RAG stack duplicates the already-dropped RAG layer - a shared UI cuts across the tenant boundary the virtual key defines All three diagrams now show tenant applications as muted context calling the gateway, rather than a bundled UI inside the platform. Kept as a candidate optional catalog block a team orders with its own virtual key, which is recorded as an open question along with the rationale. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/stackit-ai/README.md | 58 ++++-- .../stackit-ai/stackit-ai-oneclick.dot | 13 +- .../stackit-ai/stackit-ai-oneclick.svg | 82 ++++---- .../stackit-ai/stackit-ai-pluggable.dot | 13 +- .../stackit-ai/stackit-ai-pluggable.svg | 190 +++++++++--------- .../stackit-ai/stackit-ai.dot | 10 +- .../stackit-ai/stackit-ai.svg | 90 ++++----- 7 files changed, 242 insertions(+), 214 deletions(-) diff --git a/reference-architectures/stackit-ai/README.md b/reference-architectures/stackit-ai/README.md index 1de5c461..9b0ce5dd 100644 --- a/reference-architectures/stackit-ai/README.md +++ b/reference-architectures/stackit-ai/README.md @@ -1,9 +1,9 @@ --- name: STACKIT AI description: > - An opinionated, one-click AI platform: OpenWebUI, LiteLLM and Langfuse installed into a Kubernetes - namespace landing zone, with LiteLLM registered as a meshStack platform so application teams order - governed model access β€” budget, allowed models and tracing included β€” as a self-service item. + An opinionated, one-click AI platform: LiteLLM and Langfuse installed into a Kubernetes namespace + landing zone, with LiteLLM registered as a meshStack platform so application teams order governed + model access β€” budget, allowed models and tracing included β€” as a self-service item. cloudProviders: - stackit buildingBlocks: @@ -20,23 +20,24 @@ buildingBlocks: Enterprise AI adoption stalls on three questions a raw model endpoint does not answer: *who may call which model*, *what did it cost*, and *what exactly was sent and returned*. This reference architecture answers them with opinionated defaults rather than a toolkit: one order installs the -gateway, chat UI and observability stack, and a second turns model access into a governed, -self-service catalog item. +gateway and observability stack, and a second turns model access into a governed, self-service +catalog item. **Target audience:** - **Platform engineers** who want to offer LLM access as a governed product β€” per-team keys, budgets and model allow-lists β€” instead of handing out one shared credential. -- **Application teams** who need a stable OpenAI-compatible endpoint and a chat UI without operating - model infrastructure. +- **Application teams** who need a stable, governed OpenAI-compatible endpoint to build on β€” chat + interfaces, agents, assistants β€” without operating model infrastructure. ## Architecture -The **AI platform** runs on SKE inside STACKIT. **OpenWebUI** is what users see; every call goes -through **LiteLLM**, the single choke point where virtual keys, budgets and model allow-lists are +The **AI platform** runs on SKE inside STACKIT and is deliberately just two components. Every call +goes through **LiteLLM**, the single choke point where virtual keys, budgets and model allow-lists are enforced, and which routes inference to **STACKIT AI Model Serving**. **Langfuse** traces every call for evaluation and usage attribution. Self-hosted **vLLM** is shown muted β€” an optional backend, not -required when the managed sovereign API is used. +required when the managed sovereign API is used. Tenant applications are muted too: what teams build +on the endpoint is their business, not part of the platform. ![STACKIT AI reference architecture](stackit-ai.svg) @@ -47,10 +48,9 @@ into one platform-team order and one application-team order. ![One-click delivery model](stackit-ai-oneclick.svg) -**β‘  Platform team, once.** An `ai-platform` building block deploys OpenWebUI, LiteLLM and Langfuse by -Helm into a tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by -convention: LiteLLM points at Langfuse for tracing, OpenWebUI points at LiteLLM, model backends are -registered from the credentials it was given. +**β‘  Platform team, once.** An `ai-platform` building block deploys LiteLLM and Langfuse by Helm into a +tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by convention: LiteLLM +points at Langfuse for tracing, and model backends are registered from the credentials it was given. **β‘‘ Application team, per team.** LiteLLM is then registered as a **meshStack platform**, so ordering model access is a normal self-service action: the landing zone carries the policy (allowed models, @@ -73,6 +73,25 @@ itself as a **custom** platform type, with the actual tenant provisioning done b [`stackit/project`](../../modules/stackit/project) building block. A LiteLLM platform would follow the same shape, with a `litellm/team` building block in place of `stackit/project`. +### Why No Chat UI in the Core + +The platform is the **governed API**, not an end-user product. A bundled UI such as OpenWebUI was +considered and deliberately left out: + +- It is an **application, not plumbing**. LiteLLM and Langfuse are what every AI workload needs; + a chat UI is one specific product built *on* them β€” and teams will build their own. +- It brings its own **user, group and per-group model permissions**, a second policy store competing + with LiteLLM. "Who may call which model" must have exactly one home, and that home is the landing + zone plus the virtual key. +- Its built-in RAG stack duplicates the already-dropped RAG layer. +- A shared UI cuts across the tenant boundary the virtual key defines, forcing its user list to be + reconciled against meshStack projects. + +The counter-argument is real β€” a URL you can chat at beats an API key for demos and for business +users who will never write a client. That is why it stays a candidate *optional* catalog block a team +orders into its own namespace with its own key, making it the first example consumer of the platform +rather than part of it. See open question 5. + ## Pluggability: Two Independent Seams The demo stack was built so infrastructure and model serving are replaceable. That generalises into @@ -124,7 +143,9 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one `litellm/team` block would drive LiteLLM's admin API. Needs confirming. 4. **Metering.** The custom platform type accepts metering configuration; whether LiteLLM spend can feed meshStack chargeback is unresolved. -5. **Tenancy of the UI.** One shared OpenWebUI with per-user keys, or one instance per team? +5. **Optional chat UI.** A ready-made UI such as OpenWebUI is valuable for demos and for business + users who will not build their own client. Should it ship as an optional catalog block teams order + into their own namespace with their own virtual key? 6. **Dropped from the demo.** FlowiseAI (agent/workflow builder) and RAGFlow (RAG and data layer) were left out to keep the focus on serving, observability and governance. Follow-up architecture? @@ -147,7 +168,7 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one | Responsibility | Platform Team | Application Team | |-------------------------------------------------------------|:---:|:---:| | Operate the Kubernetes cluster hosting the platform | βœ… | ❌ | -| Install and upgrade OpenWebUI, LiteLLM and Langfuse | βœ… | ❌ | +| Install and upgrade LiteLLM and Langfuse | βœ… | ❌ | | Decide which models are offered and to whom | βœ… | ❌ | | Define landing zones: allowed models, budgets, rate limits | βœ… | ❌ | | Register and maintain building block definitions | βœ… | ❌ | @@ -160,7 +181,10 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one -- Demo components in scope: OpenWebUI, LiteLLM, Langfuse, STACKIT AI Model Serving. +- Demo components in scope: LiteLLM, Langfuse, STACKIT AI Model Serving. +- OpenWebUI was dropped from the core: it is an application, not platform plumbing, and its built-in + user/group model permissions would be a second policy store competing with LiteLLM. Its RAG half + also duplicates the already-dropped RAGFlow. Kept as a candidate optional block (see open question 5). - Original demo ran on Scaleway; only the runtime and model-serving layers need swapping. - Hub modules still missing for the platform components themselves β€” only `stackit/model-serving` is scaffolded, as a minimal first cut around `stackit_modelserving_token` (the only AI-specific diff --git a/reference-architectures/stackit-ai/stackit-ai-oneclick.dot b/reference-architectures/stackit-ai/stackit-ai-oneclick.dot index b833b38f..be12b461 100644 --- a/reference-architectures/stackit-ai/stackit-ai-oneclick.dot +++ b/reference-architectures/stackit-ai/stackit-ai-oneclick.dot @@ -24,7 +24,7 @@ digraph stackit_ai_oneclick { BBD_STACK [label="πŸ“¦ ai-platform BBD\nopinionated Helm wiring" fillcolor="#ecedfb" color="#9aa2e6"] NS [label="☸️ Tenant namespace\nfrom the Kubernetes landing zone" fillcolor="#e5f2ea" color="#85bfa0"] - STACK [label="πŸšͺ OpenWebUI Β· LiteLLM Β· Langfuse\nwired together by convention" fillcolor="#e5f2ea" color="#85bfa0"] + STACK [label="πŸšͺ LiteLLM Β· Langfuse\nwired together by convention" fillcolor="#e5f2ea" color="#85bfa0"] PLATFORM [label="πŸ›°οΈ LiteLLM as meshStack platform\ncustom platform type" fillcolor="#ecedfb" color="#9aa2e6"] LZAI [label="πŸ›¬ AI landing zones\nallowed models Β· budget tier" fillcolor="#ecedfb" color="#9aa2e6"] @@ -45,12 +45,13 @@ digraph stackit_ai_oneclick { BBD_TEAM [label="πŸ“¦ litellm/team BBD" fillcolor="#ecedfb" color="#9aa2e6"] KEY [label="πŸ”‘ LiteLLM team and virtual key Γ—N\nmodels and budget from the landing zone" fillcolor="#e5f2ea" color="#85bfa0"] - USER [label="πŸ‘€ Application team\nchats in OpenWebUI" fillcolor="#ffffff" color="#a2abb8"] + APPS [label="πŸ’¬ Own applications\nchat UIs, agents, assistants" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] - { rank=same; KEY -> USER [style=invis] } + { rank=same; KEY -> APPS [style=invis] } - BBD_TEAM -> KEY [xlabel="creates"] - KEY -> USER [xlabel="grants access" style=dashed] + BBD_TEAM -> KEY [xlabel="creates"] + KEY -> APPS [xlabel="authorises" style=dashed] } subgraph cluster_existing { @@ -70,6 +71,6 @@ digraph stackit_ai_oneclick { K8SLZ -> NS [xlabel="provides" style=dotted constraint=false] LZAI -> BBD_TEAM [xlabel="governs" style=dotted] - KEY -> STACK [xlabel="authorises calls" style=dashed constraint=false] + APPS -> STACK [xlabel="call the gateway API" style=dashed constraint=false] STACK -> MODELS [xlabel="routes by policy" constraint=false] } diff --git a/reference-architectures/stackit-ai/stackit-ai-oneclick.svg b/reference-architectures/stackit-ai/stackit-ai-oneclick.svg index ae887a4f..4bdb68e1 100644 --- a/reference-architectures/stackit-ai/stackit-ai-oneclick.svg +++ b/reference-architectures/stackit-ai/stackit-ai-oneclick.svg @@ -13,7 +13,7 @@ cluster_team - +  β‘‘ Application team Β· one click, per team @@ -24,37 +24,37 @@ BBD_STACK - -πŸ“¦ ai-platform BBD -opinionated Helm wiring + +πŸ“¦ ai-platform BBD +opinionated Helm wiring NS - -☸️ Tenant namespace -from the Kubernetes landing zone + +☸️ Tenant namespace +from the Kubernetes landing zone BBD_STACK->NS - - -deploys into + + +deploys into STACK - -πŸšͺ OpenWebUI Β· LiteLLM Β· Langfuse -wired together by convention + +πŸšͺ LiteLLM Β· Langfuse +wired together by convention NS->STACK - - -hosts + + +hosts @@ -80,9 +80,9 @@ STACK->MODELS - + -routes by policy +routes by policy @@ -95,8 +95,8 @@ PLATFORM->LZAI - - + + @@ -125,27 +125,27 @@ creates - - -KEY->STACK - - -authorises calls - - + -USER - -πŸ‘€ Application team -chats in OpenWebUI +APPS + +πŸ’¬ Own applications +chat UIs, agents, assistants - - + + -KEY->USER - - -grants access +KEY->APPS + + +authorises + + + +APPS->STACK + + +call the gateway API @@ -157,9 +157,9 @@ K8SLZ->NS - - -provides + + +provides diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot b/reference-architectures/stackit-ai/stackit-ai-pluggable.dot index 908a0df9..af9c2c35 100644 --- a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot +++ b/reference-architectures/stackit-ai/stackit-ai-pluggable.dot @@ -17,6 +17,8 @@ digraph stackit_ai_pluggable { edge [fontname="Helvetica" fontsize=9 fontcolor="#697180" color="#8b95a3" arrowsize=0.7] USER [label="πŸ‘€ Application team" fillcolor="#ffffff" color="#a2abb8"] + APPS [label="πŸ’¬ Tenant AI applications Γ—N\nbuilt by teams on the gateway API" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] subgraph cluster_mesh { label=" meshStack" @@ -25,7 +27,7 @@ digraph stackit_ai_pluggable { style="rounded" color="#d5dae0" margin=18 - BBD_COMP [label="πŸ“¦ openwebui Β· litellm Β· langfuse BBDs\nsupportedPlatforms: kubernetes" fillcolor="#ecedfb" color="#9aa2e6"] + BBD_COMP [label="πŸ“¦ litellm Β· langfuse BBDs\nsupportedPlatforms: kubernetes" fillcolor="#ecedfb" color="#9aa2e6"] LZ [label="πŸ›¬ Landing Zone\nprovides the namespace" fillcolor="#ecedfb" color="#9aa2e6"] BBD_MODEL [label="πŸ“¦ model-access BBD\none per provider, same output shape" fillcolor="#ecedfb" color="#9aa2e6"] @@ -39,13 +41,11 @@ digraph stackit_ai_pluggable { style="rounded" color="#d5dae0" margin=18 - WEBUI [label="πŸ’¬ OpenWebUI" fillcolor="#e5f2ea" color="#85bfa0"] LITELLM [label="πŸšͺ LiteLLM\nvirtual keys Β· budgets Β· allow-list" fillcolor="#e5f2ea" color="#85bfa0"] LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation" fillcolor="#e5f2ea" color="#85bfa0"] - { rank=same; WEBUI -> LITELLM -> LANGFUSE [style=invis] } + { rank=same; LITELLM -> LANGFUSE [style=invis] } - WEBUI -> LITELLM [xlabel="completions"] LITELLM -> LANGFUSE [xlabel="traces" style=dashed constraint=false] } @@ -79,9 +79,10 @@ digraph stackit_ai_pluggable { { rank=same; STACKITM -> AZUREM -> VLLM [style=invis] } } - USER -> WEBUI [xlabel="chats"] + USER -> APPS [xlabel="builds and uses"] + APPS -> LITELLM [xlabel="calls the gateway"] - BBD_COMP -> WEBUI [xlabel="deploys via Helm" style=dotted lhead=cluster_platform] + BBD_COMP -> LITELLM [xlabel="deploys via Helm" style=dotted lhead=cluster_platform] LZ -> SKE [xlabel="backed by" style=dotted lhead=cluster_runtime constraint=false] BBD_MODEL -> LITELLM [xlabel="virtual key and model list" style=dotted constraint=false] diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg b/reference-architectures/stackit-ai/stackit-ai-pluggable.svg index 07dc8233..d114ac6b 100644 --- a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg +++ b/reference-architectures/stackit-ai/stackit-ai-pluggable.svg @@ -1,178 +1,178 @@ - - + + stackit_ai_pluggable - + cluster_mesh - -  meshStack + +  meshStack cluster_platform - -  AI Platform Β· Helm into a tenant namespace + +  AI Platform Β· Helm into a tenant namespace cluster_runtime - -  Runtime seam Β· any meshStack Kubernetes platform + +  Runtime seam Β· any meshStack Kubernetes platform cluster_models - -  Model seam Β· OpenAI-compatible backends + +  Model seam Β· OpenAI-compatible backends USER - -πŸ‘€ Application team + +πŸ‘€ Application team - - -WEBUI - -πŸ’¬ OpenWebUI + + +APPS + +πŸ’¬ Tenant AI applications Γ—N +built by teams on the gateway API + + + +USER->APPS + + +builds and uses - - -USER->WEBUI - - -chats + + +LITELLM + +πŸšͺ LiteLLM +virtual keys Β· budgets Β· allow-list + + + +APPS->LITELLM + + +calls the gateway - + BBD_COMP - -πŸ“¦ openwebui Β· litellm Β· langfuse BBDs -supportedPlatforms: kubernetes + +πŸ“¦ litellm Β· langfuse BBDs +supportedPlatforms: kubernetes - + LZ - -πŸ›¬ Landing Zone -provides the namespace + +πŸ›¬ Landing Zone +provides the namespace - - -BBD_COMP->WEBUI - - -deploys via Helm + + +BBD_COMP->LITELLM + + +deploys via Helm - + BBD_MODEL - -πŸ“¦ model-access BBD -one per provider, same output shape + +πŸ“¦ model-access BBD +one per provider, same output shape SKE - -☸️ STACKIT SKE -sovereign + +☸️ STACKIT SKE +sovereign - + LZ->SKE - - -backed by - - - -LITELLM - -πŸšͺ LiteLLM -virtual keys Β· budgets Β· allow-list + + +backed by - + BBD_MODEL->LITELLM - - -virtual key and model list - - - - -WEBUI->LITELLM - - -completions + + +virtual key and model list LANGFUSE - -πŸ“ˆ Langfuse -tracing Β· evaluation + +πŸ“ˆ Langfuse +tracing Β· evaluation - + LITELLM->LANGFUSE - - -traces + + +traces STACKITM - -🧠 STACKIT AI Model Serving -sovereign + +🧠 STACKIT AI Model Serving +sovereign - + LITELLM->STACKITM - - -routes by policy + + +routes by policy - + LANGFUSE->SKE - - -namespace runs on + + +namespace runs on AKS - -☸️ Azure AKS + +☸️ Azure AKS ANYK8S - -☸️ Any conformant cluster + +☸️ Any conformant cluster AZUREM - -🧠 Azure OpenAI + +🧠 Azure OpenAI VLLM - -🧠 Self-hosted vLLM + +🧠 Self-hosted vLLM diff --git a/reference-architectures/stackit-ai/stackit-ai.dot b/reference-architectures/stackit-ai/stackit-ai.dot index 6039d5cf..d7ce3a80 100644 --- a/reference-architectures/stackit-ai/stackit-ai.dot +++ b/reference-architectures/stackit-ai/stackit-ai.dot @@ -30,7 +30,6 @@ digraph stackit_ai { style="rounded" color="#d5dae0" margin=14 - WEBUI [label="πŸ’¬ OpenWebUI\nuser-facing chat interface" fillcolor="#e5f2ea" color="#85bfa0"] LITELLM [label="πŸšͺ LiteLLM\nvirtual keys Β· budgets Β· model allow-list" fillcolor="#e5f2ea" color="#85bfa0"] LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation Β· usage" fillcolor="#e5f2ea" color="#85bfa0"] VLLM [label="🧠 vLLM\nself-hosted models Β· optional" @@ -38,14 +37,17 @@ digraph stackit_ai { { rank=same; LITELLM -> LANGFUSE [style=invis] } - WEBUI -> LITELLM [xlabel="completions"] LITELLM -> LANGFUSE [xlabel="traces every call" style=dashed constraint=false] LITELLM -> VLLM [xlabel="optional backend" style=dashed] } + APPS [label="πŸ’¬ Tenant AI applications Γ—N\nchat UIs, agents, assistants β€” built by teams" + fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] + MODEL [label="🧠 STACKIT AI Model Serving\nmanaged sovereign LLM API" fillcolor="#eef2f6" color="#93a7bb"] - LITELLM -> MODEL [xlabel="routes inference to"] + APPS -> LITELLM [xlabel="calls the gateway"] + LITELLM -> MODEL [xlabel="routes inference to"] } subgraph cluster_mesh { @@ -67,7 +69,7 @@ digraph stackit_ai { BBD_M -> PROJ [xlabel="ordered in" constraint=false] } - USER -> WEBUI [xlabel="chats"] + USER -> APPS [xlabel="builds and uses"] BBD_M -> LITELLM [xlabel="issues virtual key" style=dotted constraint=false] BBD_M -> MODEL [xlabel="issues serving token" style=dotted constraint=false] LANGFUSE -> PROJ [xlabel="usage attribution" style=dotted constraint=false] diff --git a/reference-architectures/stackit-ai/stackit-ai.svg b/reference-architectures/stackit-ai/stackit-ai.svg index e67258b0..9ef1b157 100644 --- a/reference-architectures/stackit-ai/stackit-ai.svg +++ b/reference-architectures/stackit-ai/stackit-ai.svg @@ -1,62 +1,55 @@ - - + + stackit_ai - + cluster_stackit - -  STACKIT β€” sovereign infrastructure + +  STACKIT β€” sovereign infrastructure cluster_stack - -  AI Platform Β· deployed on SKE + +  AI Platform Β· deployed on SKE cluster_mesh - -  meshStack β€” self-service and governance + +  meshStack β€” self-service and governance USER - -πŸ‘€ Application team + +πŸ‘€ Application team - - -WEBUI - -πŸ’¬ OpenWebUI -user-facing chat interface + + +APPS + +πŸ’¬ Tenant AI applications Γ—N +chat UIs, agents, assistants β€” built by teams - + -USER->WEBUI - - -chats +USER->APPS + + +builds and uses - + LITELLM πŸšͺ LiteLLM virtual keys Β· budgets Β· model allow-list - - -WEBUI->LITELLM - - -completions - - + LANGFUSE πŸ“ˆ Langfuse @@ -64,21 +57,21 @@ - + LITELLM->LANGFUSE - - -traces every call + + +traces every call - + VLLM 🧠 vLLM self-hosted models Β· optional - + LITELLM->VLLM @@ -112,11 +105,18 @@ usage attribution + + +APPS->LITELLM + + +calls the gateway + PLAT - -πŸ›°οΈ SKE Platform + +πŸ›°οΈ SKE Platform @@ -128,8 +128,8 @@ PLAT->LZ - - + + @@ -149,9 +149,9 @@ BBD_M->LITELLM - - -issues virtual key + + +issues virtual key From 2fb6b2129db19bf5be8b7044804c327c32a2fff1 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 09:44:19 +0200 Subject: [PATCH 04/60] docs(ref-arch): track folding the SKE Starterkit into the AI architecture The starterkit demo app already calls an AI model, but its credential is injected statically via a foundation-local ai.tf. Records the finding that the integration seam already exists in the right shape: forgejo-connector additional_kubernetes_secrets carries base URL, API key and model name, so pointing those at LiteLLM needs no interface change. Also records the delivery idea of exposing AI as an opt-in option following the stackit-landingzone nullable-object pattern, with the open layering question that the STACKIT LZ hands out projects while the AI platform needs a namespace. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/stackit-ai/README.md | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/reference-architectures/stackit-ai/README.md b/reference-architectures/stackit-ai/README.md index 9b0ce5dd..801c3830 100644 --- a/reference-architectures/stackit-ai/README.md +++ b/reference-architectures/stackit-ai/README.md @@ -149,6 +149,37 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one 6. **Dropped from the demo.** FlowiseAI (agent/workflow builder) and RAGFlow (RAG and data layer) were left out to keep the focus on serving, observability and governance. Follow-up architecture? +## Tracked: Folding In the SKE Starterkit + +Not in scope for the first iteration, recorded so it is not lost. + +The [`ske/ske-starterkit`](../../modules/ske/ske-starterkit) demo app already calls an AI model, but +its credential is injected statically β€” foundations supply a STACKIT model-serving token through +their own `ai.tf`. The goal is for the starterkit to consume this reference architecture instead, so +its demo app becomes a live example of a governed AI workload. + +**The seam already exists and is already the right shape.** The starterkit injects AI config through +`forgejo-connector`'s `additional_kubernetes_secrets` as a `stackit-ai` secret holding three values: + +| Variable | Today | With this architecture | +|----------|-------|------------------------| +| `STACKIT_AI_BASE_URL` | STACKIT Model Serving endpoint | The LiteLLM gateway URL | +| `STACKIT_AI_API_KEY` | A statically provisioned token | The team's LiteLLM virtual key | +| `STACKIT_AI_MODEL` | A fixed model name | A model from the landing zone's allow-list | + +Because LiteLLM is OpenAI-compatible, this needs **no change to the starterkit's interface** β€” only +different values. Two notes: the `STACKIT_AI_*` prefix becomes a misnomer once the gateway fronts +several providers (renaming to `OPENAI_BASE_URL` / `OPENAI_API_KEY` would additionally make most +client SDKs work with zero configuration), and routing the demo app through the gateway means its +traffic shows up in Langfuse and counts against the team's budget β€” which is the point. + +**Delivery idea to validate:** expose AI as an opt-in option the same way +[`stackit-landingzone`](../stackit-landingzone) exposes networking β€” a nullable object variable +(`variable "network"`, unset = sandbox only) β€” with the SKE Starterkit as a further option that +requires the AI option to be enabled. Open layering question: the STACKIT Landing Zone hands out +STACKIT **projects**, while the AI platform needs a Kubernetes **namespace**, so the option may +belong at the SKE/Kubernetes layer instead. + ## Getting Started ### Prerequisites From 854d238cbfcd82d468a9b1a6679472664c5350a7 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 10:07:30 +0200 Subject: [PATCH 05/60] docs(ref-arch): record install delivery decision and the BBD platform-ref gap Step 1 (platform-team install) is delivered by the reference architecture own Terraform apply rather than a catalog building block, so the target SKE cluster is a plain input variable and instance selection works today. Records the underlying limitation as a meshStack feature request: BBD supported_platforms.kind is Always meshPlatformType for now, while landingzone.platform_ref already targets a platform by uuid. Instance-level support would make the catalog path pluggable, which matters for the realistic two-cluster topology (shared AI platform vs application workloads). Also corrects the SKE Starterkit note: no layering conflict, since the SKE cluster is itself an offering inside a STACKIT project. Notes that the cluster building block does not exist in the hub yet. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/stackit-ai/README.md | 40 +++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/reference-architectures/stackit-ai/README.md b/reference-architectures/stackit-ai/README.md index 801c3830..8c05dee2 100644 --- a/reference-architectures/stackit-ai/README.md +++ b/reference-architectures/stackit-ai/README.md @@ -52,9 +52,15 @@ into one platform-team order and one application-team order. tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by convention: LiteLLM points at Langfuse for tracing, and model backends are registered from the credentials it was given. +Step β‘  is performed by the reference architecture's **own Terraform apply** rather than by ordering a +catalog building block β€” the same way `stackit-landingzone` provisions its platform and landing zone +directly. This matters: the target cluster is then a plain input variable, so selecting *which* SKE +cluster the AI platform lands in works today (see [Feature Requests](#meshstack-feature-requests)). + **β‘‘ Application team, per team.** LiteLLM is then registered as a **meshStack platform**, so ordering model access is a normal self-service action: the landing zone carries the policy (allowed models, -budget tier), and the building block creates the LiteLLM team and virtual key behind it. +budget tier), and the building block creates the LiteLLM team and virtual key behind it. This step is +per-project and needs no cluster targeting, so it is unaffected by the limitation below. ### Why LiteLLM Works as a meshStack Platform @@ -149,6 +155,25 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one 6. **Dropped from the demo.** FlowiseAI (agent/workflow builder) and RAGFlow (RAG and data layer) were left out to keep the focus on serving, observability and governance. Follow-up architecture? +## meshStack Feature Requests + +**Instance-level `supported_platforms` on building block definitions.** Today +`meshstack_building_block_definition.supported_platforms.kind` is documented as *"Always +`meshPlatformType` for now"* β€” a block can declare it supports `kubernetes`, but not *which* +Kubernetes platform. By contrast `meshstack_landingzone.platform_ref` already targets a specific +platform by uuid. + +This is why step β‘  is RA-owned Terraform rather than a catalog block. Allowing `supported_platforms` +to reference a `meshPlatform` would make the catalog path pluggable and is worth having: + +- The realistic topology is **two SKE clusters** β€” one hosting the shared AI platform, one hosting + application workloads such as the SKE Starterkit. Without instance-level support, an `ai-platform` + block offered for type `kubernetes` appears orderable on both. +- It would let the platform engineer answer "which cluster?" as a normal platform reference instead of + threading a kubeconfig or platform uuid through Terraform inputs. +- The current workaround β€” registering each cluster as its own custom platform type β€” inflates the + platform-type list to express what is really an instance selection. + ## Tracked: Folding In the SKE Starterkit Not in scope for the first iteration, recorded so it is not lost. @@ -173,12 +198,17 @@ several providers (renaming to `OPENAI_BASE_URL` / `OPENAI_API_KEY` would additi client SDKs work with zero configuration), and routing the demo app through the gateway means its traffic shows up in Langfuse and counts against the team's budget β€” which is the point. -**Delivery idea to validate:** expose AI as an opt-in option the same way +**Delivery idea:** expose AI as an opt-in option the same way [`stackit-landingzone`](../stackit-landingzone) exposes networking β€” a nullable object variable (`variable "network"`, unset = sandbox only) β€” with the SKE Starterkit as a further option that -requires the AI option to be enabled. Open layering question: the STACKIT Landing Zone hands out -STACKIT **projects**, while the AI platform needs a Kubernetes **namespace**, so the option may -belong at the SKE/Kubernetes layer instead. +requires the AI option to be enabled. + +There is no layering conflict here, because the **SKE cluster is itself an offering inside a STACKIT +project**: the STACKIT LZ provisions the project, the SKE cluster building block turns it into a +Kubernetes platform, and that platform's landing zone hands out the namespaces the AI components and +the starterkit need. Note the cluster building block does not exist in the hub yet β€” SKE clusters are +provisioned by foundation Terraform today (`platforms/ske/kubernetes/cluster.tf` in the +cloudfoundation repos), so hub-ifying it is a prerequisite. ## Getting Started From 5fd331e8fc8cd0ec5db91d64bde01c757aa141ec Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 10:10:51 +0200 Subject: [PATCH 06/60] refactor(ref-arch): generalise to a multi-cloud AI platform architecture Both pluggable seams are real, so this is one capability-scoped reference architecture rather than a STACKIT fork and an Azure fork. Renames stackit-ai to ai-platform, lists stackit and azure as cloudProviders, and frames STACKIT as the sovereign reference instantiation. Cloud-agnostic components are destined for a new modules/ai/ namespace (ai/litellm, ai/langfuse, ai/litellm-team); each provider contributes only a small model-access module, of which stackit/model-serving is the first. Extends the reference-architecture folder-name convention in AGENTS.md to allow a capability-only name for genuinely multi-cloud architectures. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 4 ++- .../{stackit-ai => ai-platform}/README.md | 33 +++++++++++-------- .../ai-platform-oneclick.dot} | 2 +- .../ai-platform-oneclick.svg} | 2 +- .../ai-platform-pluggable.dot} | 2 +- .../ai-platform-pluggable.svg} | 2 +- .../ai-platform.dot} | 4 +-- .../ai-platform.svg} | 2 +- 8 files changed, 29 insertions(+), 22 deletions(-) rename reference-architectures/{stackit-ai => ai-platform}/README.md (92%) rename reference-architectures/{stackit-ai/stackit-ai-oneclick.dot => ai-platform/ai-platform-oneclick.dot} (99%) rename reference-architectures/{stackit-ai/stackit-ai-oneclick.svg => ai-platform/ai-platform-oneclick.svg} (99%) rename reference-architectures/{stackit-ai/stackit-ai-pluggable.dot => ai-platform/ai-platform-pluggable.dot} (99%) rename reference-architectures/{stackit-ai/stackit-ai-pluggable.svg => ai-platform/ai-platform-pluggable.svg} (99%) rename reference-architectures/{stackit-ai/stackit-ai.dot => ai-platform/ai-platform.dot} (96%) rename reference-architectures/{stackit-ai/stackit-ai.svg => ai-platform/ai-platform.svg} (99%) diff --git a/AGENTS.md b/AGENTS.md index 5c36f59d..07ac7d25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -338,7 +338,9 @@ getting-started steps, and shared responsibility matrix. ### Conventions - Folder name: `-` (e.g. `azure-kubernetes`, `stackit-kubernetes`), with the - architecture itself in `README.md`. + architecture itself in `README.md`. Use `` alone (e.g. `ai-platform`) when the + architecture is genuinely multi-cloud β€” it lists several `cloudProviders` and its components are + cloud-agnostic, with each provider contributing only a small provider-specific module. - Logo: colocate as `logo.png` (or `logo.svg`) in the architecture folder, the same convention `buildingblock/logo.png` uses. The website generator copies it to `website/public/assets/reference-architecture-logos/.png` β€” never add files there by hand, diff --git a/reference-architectures/stackit-ai/README.md b/reference-architectures/ai-platform/README.md similarity index 92% rename from reference-architectures/stackit-ai/README.md rename to reference-architectures/ai-platform/README.md index 8c05dee2..81d8c35a 100644 --- a/reference-architectures/stackit-ai/README.md +++ b/reference-architectures/ai-platform/README.md @@ -1,17 +1,19 @@ --- -name: STACKIT AI +name: AI Platform description: > An opinionated, one-click AI platform: LiteLLM and Langfuse installed into a Kubernetes namespace landing zone, with LiteLLM registered as a meshStack platform so application teams order governed - model access β€” budget, allowed models and tracing included β€” as a self-service item. + model access β€” budget, allowed models and tracing included β€” as a self-service item. The runtime and + the model backend are both pluggable; STACKIT is the sovereign reference instantiation. cloudProviders: - stackit + - azure buildingBlocks: - path: stackit/model-serving role: Issues the STACKIT AI Model Serving credential that LiteLLM routes sovereign inference through. --- -# STACKIT AI +# AI Platform ## Overview @@ -32,21 +34,22 @@ catalog item. ## Architecture -The **AI platform** runs on SKE inside STACKIT and is deliberately just two components. Every call +The diagram below shows the **STACKIT instantiation** β€” the sovereign reference. The platform runs on +SKE and is deliberately just two components. Every call goes through **LiteLLM**, the single choke point where virtual keys, budgets and model allow-lists are enforced, and which routes inference to **STACKIT AI Model Serving**. **Langfuse** traces every call for evaluation and usage attribution. Self-hosted **vLLM** is shown muted β€” an optional backend, not required when the managed sovereign API is used. Tenant applications are muted too: what teams build on the endpoint is their business, not part of the platform. -![STACKIT AI reference architecture](stackit-ai.svg) +![AI Platform reference architecture](ai-platform.svg) ## Delivery Model: Two Orders The architecture is deliberately opinionated so it is ready to go rather than assembled. It splits into one platform-team order and one application-team order. -![One-click delivery model](stackit-ai-oneclick.svg) +![One-click delivery model](ai-platform-oneclick.svg) **β‘  Platform team, once.** An `ai-platform` building block deploys LiteLLM and Langfuse by Helm into a tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by convention: LiteLLM @@ -101,16 +104,19 @@ rather than part of it. See open question 5. ## Pluggability: Two Independent Seams The demo stack was built so infrastructure and model serving are replaceable. That generalises into -**two orthogonal seams** β€” and because they are orthogonal, this should stay *one* reference -architecture rather than forking into a STACKIT and an Azure variant. Running on SKE while calling -Azure OpenAI, or on AKS while calling STACKIT, are both valid combinations. +**two orthogonal seams**, and because they are orthogonal this is *one* reference architecture rather +than a STACKIT and an Azure fork. Running on SKE while calling Azure OpenAI, or on AKS while calling +STACKIT, are both valid combinations. + +This is why the architecture is named for the capability rather than a cloud: cloud-agnostic +components live in `modules/ai/`, and each provider contributes only a small model-access module. | Seam | Contract | Chosen by | Implementations | |------|----------|-----------|-----------------| | **Runtime** β€” where the components run | A landing zone that hands out Kubernetes namespaces; the blocks declare `supportedPlatforms: kubernetes` and never name a cloud | The landing zone the platform team orders into | STACKIT SKE, Azure AKS, any conformant cluster | | **Model** β€” where inference happens | An OpenAI-compatible endpoint plus credential, surfaced as a LiteLLM `model_list` entry | LiteLLM routing policy, fed by one model-access block per provider | `stackit/model-serving`, an Azure OpenAI equivalent, self-hosted vLLM | -![Pluggable seams variant](stackit-ai-pluggable.svg) +![Pluggable seams variant](ai-platform-pluggable.svg) Adding a cloud therefore means adding one small model-access module with the same output shape β€” not changing the architecture. The runtime seam needs no per-cloud work at all: the precedent is @@ -138,10 +144,9 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one -1. **Scope and name.** If both seams are real, this is an *AI platform* reference architecture with - `cloudProviders: [stackit, azure]`, not a STACKIT-only one β€” STACKIT would be the sovereign - reference instantiation. That implies renaming the folder and finding a home for the - runtime-agnostic modules outside `modules/stackit/`. +1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong + in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing + `ai/litellm-team` β€” none of which are written yet. 2. **Bootstrap ordering.** The LiteLLM platform can only be registered once LiteLLM is reachable (URL plus admin credential). `stackit-landingzone` already registers a platform and orders a building block instance in one apply, so there is a precedent β€” but the dependency needs designing. diff --git a/reference-architectures/stackit-ai/stackit-ai-oneclick.dot b/reference-architectures/ai-platform/ai-platform-oneclick.dot similarity index 99% rename from reference-architectures/stackit-ai/stackit-ai-oneclick.dot rename to reference-architectures/ai-platform/ai-platform-oneclick.dot index be12b461..4bf94bed 100644 --- a/reference-architectures/stackit-ai/stackit-ai-oneclick.dot +++ b/reference-architectures/ai-platform/ai-platform-oneclick.dot @@ -3,7 +3,7 @@ * custom meshStack platform so model access itself becomes a landing-zone-governed order. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ -digraph stackit_ai_oneclick { +digraph ai_platform_oneclick { rankdir=TB splines=ortho forcelabels=true diff --git a/reference-architectures/stackit-ai/stackit-ai-oneclick.svg b/reference-architectures/ai-platform/ai-platform-oneclick.svg similarity index 99% rename from reference-architectures/stackit-ai/stackit-ai-oneclick.svg rename to reference-architectures/ai-platform/ai-platform-oneclick.svg index 4bdb68e1..ca365450 100644 --- a/reference-architectures/stackit-ai/stackit-ai-oneclick.svg +++ b/reference-architectures/ai-platform/ai-platform-oneclick.svg @@ -4,7 +4,7 @@ -stackit_ai_oneclick +ai_platform_oneclick cluster_once diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot b/reference-architectures/ai-platform/ai-platform-pluggable.dot similarity index 99% rename from reference-architectures/stackit-ai/stackit-ai-pluggable.dot rename to reference-architectures/ai-platform/ai-platform-pluggable.dot index af9c2c35..1473624c 100644 --- a/reference-architectures/stackit-ai/stackit-ai-pluggable.dot +++ b/reference-architectures/ai-platform/ai-platform-pluggable.dot @@ -3,7 +3,7 @@ * provides the namespace, and the model backends LiteLLM routes to. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ -digraph stackit_ai_pluggable { +digraph ai_platform_pluggable { rankdir=TB compound=true splines=ortho diff --git a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg b/reference-architectures/ai-platform/ai-platform-pluggable.svg similarity index 99% rename from reference-architectures/stackit-ai/stackit-ai-pluggable.svg rename to reference-architectures/ai-platform/ai-platform-pluggable.svg index d114ac6b..1936b2af 100644 --- a/reference-architectures/stackit-ai/stackit-ai-pluggable.svg +++ b/reference-architectures/ai-platform/ai-platform-pluggable.svg @@ -4,7 +4,7 @@ -stackit_ai_pluggable +ai_platform_pluggable cluster_mesh diff --git a/reference-architectures/stackit-ai/stackit-ai.dot b/reference-architectures/ai-platform/ai-platform.dot similarity index 96% rename from reference-architectures/stackit-ai/stackit-ai.dot rename to reference-architectures/ai-platform/ai-platform.dot index d7ce3a80..a444c7cc 100644 --- a/reference-architectures/stackit-ai/stackit-ai.dot +++ b/reference-architectures/ai-platform/ai-platform.dot @@ -1,8 +1,8 @@ /* - * STACKIT AI reference architecture β€” governed, observable sovereign model serving. + * AI Platform reference architecture β€” governed, observable sovereign model serving. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ -digraph stackit_ai { +digraph ai_platform { rankdir=TB splines=ortho forcelabels=true diff --git a/reference-architectures/stackit-ai/stackit-ai.svg b/reference-architectures/ai-platform/ai-platform.svg similarity index 99% rename from reference-architectures/stackit-ai/stackit-ai.svg rename to reference-architectures/ai-platform/ai-platform.svg index 9ef1b157..023349cc 100644 --- a/reference-architectures/stackit-ai/stackit-ai.svg +++ b/reference-architectures/ai-platform/ai-platform.svg @@ -4,7 +4,7 @@ -stackit_ai +ai_platform cluster_stackit From 9190999b58f8a0e08728c4e42f17a3b674713ea0 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 10:15:51 +0200 Subject: [PATCH 07/60] docs(ref-arch): decide per-tenant model serving token scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each tenant gets its own STACKIT Model Serving token in its own STACKIT project rather than LiteLLM holding one shared platform credential. Recorded as an opinionated call, with the shared-token variant noted as legitimate for other platform engineering setups. Rationale: keeps the STACKIT resource hierarchy reflecting tenant structure, puts spend on the tenant project so it follows the existing STACKIT cost path instead of needing AI-specific attribution, and narrows rotation blast radius. Also records that app-team ordering stays one click either way β€” the added complexity is inside the block, which must touch both STACKIT and the LiteLLM admin API, bringing two-system consistency and rotation concerns to design for. This validates the tenant-level shape of the scaffolded module. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index 81d8c35a..23818787 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -126,6 +126,29 @@ that takes a kubeconfig and declares `supportedPlatforms: kubernetes`. This architecture **consumes** a Kubernetes cluster, it does not provision one β€” which is why [`stackit-kubernetes`](../stackit-kubernetes) stays a separate, companion reference architecture. +## Token Scope: Per-Tenant + +Each tenant gets **its own STACKIT Model Serving token**, issued into its own STACKIT project, rather +than LiteLLM holding one shared platform credential. This is an opinionated call β€” the shared-token +variant is simpler and legitimate for some platform engineering setups β€” made for three reasons: + +- **Clean provider hierarchy.** The credential lives in the tenant's own STACKIT project, so the + STACKIT resource hierarchy keeps reflecting the tenant structure instead of collapsing all AI usage + onto one platform project. +- **Cost attribution without new plumbing.** Because spend lands on the tenant's own project, it flows + through whatever STACKIT cost path the platform already uses, rather than needing an AI-specific + path that reconstructs per-team spend from gateway data and pushes it back into meshStack. +- **Blast radius.** One tenant's credential can be revoked or rotated without touching anyone else. + +**Ordering stays one click.** The app team's flow is identical to the shared-token variant β€” the extra +work is inside the building block, which touches two systems: it issues the STACKIT token in the +tenant's project *and* registers the corresponding deployment, team and virtual key in LiteLLM. Two +consequences to design for: + +- **Two-system consistency.** A token created but not registered in LiteLLM leaves an orphan. The + block needs to be idempotent and to clean up on partial failure. +- **Rotation touches both.** Token TTL expiry must update the gateway too, not just STACKIT. + ## Governance and Observability @@ -151,7 +174,8 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one plus admin credential). `stackit-landingzone` already registers a platform and orders a building block instance in one apply, so there is a precedent β€” but the dependency needs designing. 3. **Provisioning mechanism.** There is no LiteLLM Terraform provider as far as we know, so the - `litellm/team` block would drive LiteLLM's admin API. Needs confirming. + `ai/litellm-team` block would drive LiteLLM's admin API β€” now on the critical path, since per-tenant + tokens mean the block must register a per-team deployment as well as the key. 4. **Metering.** The custom platform type accepts metering configuration; whether LiteLLM spend can feed meshStack chargeback is unresolved. 5. **Optional chat UI.** A ready-made UI such as OpenWebUI is valuable for demos and for business From 452e5528f65a4295f735ea26122cf1d3f8dca746 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 10:19:07 +0200 Subject: [PATCH 08/60] docs(ref-arch): make the deployment vs tenancy model explicit LiteLLM and Langfuse are each deployed once and are internally multi-tenant; nothing is deployed per application team. Tenancy is a LiteLLM team plus virtual key and a Langfuse organization/project, while the STACKIT token lives in the tenant own project. Shared deployment is forced for LiteLLM, since a gateway only enforces budgets and allow-lists if all traffic passes through one instance, and chosen for Langfuse because per-tenant installs would each carry a full data stack. States the trade-off plainly: isolation rests on the Langfuse project boundary rather than a Kubernetes or network boundary, and the shared instances are a common blast radius. Also resolves the provisioning mechanism to Mastercard/restapi, already the established pattern in this repo. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index 23818787..c70016c4 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -126,6 +126,28 @@ that takes a kubeconfig and declares `supportedPlatforms: kubernetes`. This architecture **consumes** a Kubernetes cluster, it does not provision one β€” which is why [`stackit-kubernetes`](../stackit-kubernetes) stays a separate, companion reference architecture. +## Deployment vs Tenancy + +The platform components are deployed **once** and are internally multi-tenant. Nothing is deployed per +application team: + +| Component | Deployments | Tenancy unit | Provisioned by | +|-----------|-------------|--------------|----------------| +| LiteLLM | one, shared | Team + virtual key | `ai/litellm-team` | +| Langfuse | one, shared | Organization/project + scoped API key | `ai/litellm-team` | +| STACKIT AI Model Serving | managed service | Token in the tenant's own STACKIT project | `stackit/model-serving` | + +This is forced for LiteLLM β€” a gateway only enforces budgets and allow-lists if everything goes +through one instance β€” and chosen for Langfuse, where per-tenant deployments would be disproportionate +(recent Langfuse versions need ClickHouse and Redis alongside Postgres, so each tenant install would +carry a full data stack). + +The trade-off to accept consciously: **isolation rests on Langfuse's project boundary, not on a +Kubernetes or network boundary**, and the shared instances are a common blast radius β€” if Langfuse is +down, no team has tracing. For a sovereignty story this is usually fine, since the data never leaves +the cluster; a tenant with stricter isolation requirements would need its own deployment, which this +architecture does not attempt. + ## Token Scope: Per-Tenant Each tenant gets **its own STACKIT Model Serving token**, issued into its own STACKIT project, rather @@ -173,9 +195,9 @@ consequences to design for: 2. **Bootstrap ordering.** The LiteLLM platform can only be registered once LiteLLM is reachable (URL plus admin credential). `stackit-landingzone` already registers a platform and orders a building block instance in one apply, so there is a precedent β€” but the dependency needs designing. -3. **Provisioning mechanism.** There is no LiteLLM Terraform provider as far as we know, so the - `ai/litellm-team` block would drive LiteLLM's admin API β€” now on the critical path, since per-tenant - tokens mean the block must register a per-team deployment as well as the key. +3. **Provisioning mechanism β€” resolved.** No LiteLLM or Langfuse Terraform provider exists, so + `ai/litellm-team` drives both admin APIs with `Mastercard/restapi`, already the established pattern + in this repo (13 usages), with `hashicorp/helm` for the chart deploys. No new pattern needed. 4. **Metering.** The custom platform type accepts metering configuration; whether LiteLLM spend can feed meshStack chargeback is unresolved. 5. **Optional chat UI.** A ready-made UI such as OpenWebUI is valuable for demos and for business @@ -264,7 +286,7 @@ cloudfoundation repos), so hub-ifying it is a prerequisite. | Register and maintain building block definitions | βœ… | ❌ | | Order model access from the self-service catalog | ❌ | βœ… | | Stay within the granted budget and model allow-list | ❌ | βœ… | -| Review own traces and evaluations in Langfuse | ❌ | βœ… | +| Review own traces and evaluations in their Langfuse project | ❌ | βœ… | | Build and operate the AI application or assistant | ❌ | βœ… | ## Notes for the Session From 182a9254a4d98d751580d46430f95da865f068a4 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 10:24:29 +0200 Subject: [PATCH 09/60] docs(ref-arch): resolve bootstrap ordering, request sensitive BB outputs Provider configurations may reference resource attributes but fail when the value is unknown at plan time, so the RA pre-generates the LiteLLM admin key and derives the endpoint from a known hostname. Both are knowable up front, leaving depends_on sufficient for a single apply. Adds a second meshStack feature request: encrypted sensitive outputs between building blocks would allow splitting this into two properly dependent blocks, one deploying the gateway and one registering the platform, removing the plan-time-unknown constraint entirely. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index c70016c4..d8f255b2 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -192,9 +192,11 @@ consequences to design for: 1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing `ai/litellm-team` β€” none of which are written yet. -2. **Bootstrap ordering.** The LiteLLM platform can only be registered once LiteLLM is reachable (URL - plus admin credential). `stackit-landingzone` already registers a platform and orders a building - block instance in one apply, so there is a precedent β€” but the dependency needs designing. +2. **Bootstrap ordering β€” resolved for one apply.** Provider configurations *may* reference resource + attributes, but fail when the value is unknown at plan time. So the RA generates the LiteLLM admin + key itself (`random_password`), passes it into the Helm values and derives the endpoint from a known + hostname β€” both knowable up front, leaving `depends_on` sufficient. `stackit-landingzone` already + uses `depends_on` between a platform and a building block instance in one apply. 3. **Provisioning mechanism β€” resolved.** No LiteLLM or Langfuse Terraform provider exists, so `ai/litellm-team` drives both admin APIs with `Mastercard/restapi`, already the established pattern in this repo (13 usages), with `hashicorp/helm` for the chart deploys. No new pattern needed. @@ -225,6 +227,12 @@ to reference a `meshPlatform` would make the catalog path pluggable and is worth - The current workaround β€” registering each cluster as its own custom platform type β€” inflates the platform-type list to express what is really an instance selection. +**Sensitive outputs between building blocks.** If meshStack supported encrypted sensitive outputs from +one building block into another, this architecture could be split into two properly dependent blocks β€” +one deploying the gateway, one registering the platform and consuming its endpoint and admin +credential β€” instead of relying on a pre-generated credential to keep everything in one apply. That is +the cleaner decomposition and would remove the plan-time-unknown constraint entirely. + ## Tracked: Folding In the SKE Starterkit Not in scope for the first iteration, recorded so it is not lost. From d1b1703bc90822159c0348540d061e698f490619 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 14:00:50 +0200 Subject: [PATCH 10/60] docs(ref-arch): compose AI platform as a reusable RA, consolidate cluster ingress Reframe delivery around meshStack as the Terraform runtime rather than terragrunt: the RA is itself a WORKSPACE_LEVEL building block, and the cluster credential arrives as a STATIC encrypted input read back through file() - so it applies in one pass with no two-unit split. Record that ai-platform stays generic and is reused by provider-specific RAs via git URL, with the model seam as a map plus a separate sensitive key map. Add modules/kubernetes/ingress as the TLS prerequisite, documenting the four drifting copies in the foundation repos. Correct both feature requests: WORKSPACE_LEVEL needs no supported_platforms, and sensitive inputs are excluded only for BUILDING_BLOCK_OUTPUT. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 185 +++++++++++++++--- 1 file changed, 159 insertions(+), 26 deletions(-) diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index d8f255b2..ed05b07c 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -55,16 +55,69 @@ into one platform-team order and one application-team order. tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by convention: LiteLLM points at Langfuse for tracing, and model backends are registered from the credentials it was given. -Step β‘  is performed by the reference architecture's **own Terraform apply** rather than by ordering a -catalog building block β€” the same way `stackit-landingzone` provisions its platform and landing zone -directly. This matters: the target cluster is then a plain input variable, so selecting *which* SKE -cluster the AI platform lands in works today (see [Feature Requests](#meshstack-feature-requests)). +Step β‘  is itself a **building block ordered in the platform team's own workspace** β€” the reference +architecture *is* a building block, following the shape +[`stackit-landingzone`](../stackit-landingzone) already uses: a thin `meshstack_integration.tf` +declaring one BBD whose implementation points at this architecture's `buildingblock/`, which then +creates the platform, landing zones and the tenant-facing BBDs. The meshStack instance is the +Terraform runtime; the foundation repos only instantiate the architecture. + +Two schema details make this work today, without waiting on any new meshStack feature: + +- **`target_type = "WORKSPACE_LEVEL"`** (the default) attaches the block to the platform workspace + rather than to a tenant. `supported_platforms` is required *only* for `TENANT_LEVEL`, so this step + needs no platform reference at all β€” which is why the instance-level limitation discussed under + [Feature Requests](#meshstack-feature-requests) does not block it. +- **The cluster credential arrives as an encrypted static input.** The architecture's Terraform holds + the admin kubeconfig, mints a scoped one, and bakes it into the BBD β€” exactly what + [`kubernetes/manifest`](../../modules/kubernetes/manifest) does at + `meshstack_integration.tf:159`: + + ```hcl + assignment_type = "STATIC" + sensitive = { argument = { + secret_value = "data:application/yaml;base64,${base64encode(module.backplane.kubeconfig)}" + }} + ``` + + The block then applies in **one pass**, because its provider reads `file("kubeconfig.yaml")` β€” a + value known at plan time. There is no unknown-after-apply problem to solve and no terragrunt-style + two-unit split. The consequence to accept: because the credential is static, there is one BBD per + target cluster, so cluster choice is a Terraform variable rather than an order-time selection. **β‘‘ Application team, per team.** LiteLLM is then registered as a **meshStack platform**, so ordering model access is a normal self-service action: the landing zone carries the policy (allowed models, budget tier), and the building block creates the LiteLLM team and virtual key behind it. This step is per-project and needs no cluster targeting, so it is unaffected by the limitation below. +### Composition: Architectures Reuse Architectures + +`ai-platform` is deliberately generic β€” it names no cloud and depends on neither the `stackit` nor the +`azurerm` provider. Provider-specific architectures **reuse** it and fill its two seams: + +```hcl +# reference-architectures/stackit-landingzone/buildingblock/main.tf +module "ai_platform" { + source = "github.com/meshcloud/meshstack-hub//reference-architectures/ai-platform/buildingblock?ref=${var.hub.git_ref}" + # ... SKE for the runtime seam, STACKIT Model Serving for the model seam +} +``` + +Composition must use the **git URL with `?ref=${var.hub.git_ref}`**, not a relative `../../ai-platform` +path. Relative sources appear only in `e2e/` harnesses and for directories *below* a building block's +`repository_path`; a `../` escape out of the building block's own path is not a pattern this repo +relies on. + +In `stackit-landingzone`, AI becomes an opt-in option shaped exactly like its existing +`variable "network"` β€” a nullable `object({...})` with `optional()` fields, unset meaning "sandbox +only". Enabling it plugs SKE in as the runtime and STACKIT AI Model Serving in as the model backend, +and the resulting platform is where the SKE Starterkit can then be integrated (see +[Tracked](#tracked-folding-in-the-ske-starterkit)). + +Note this is new ground: no reference architecture reuses another one yet. `stackit-landingzone`'s +integration file contains no `module` blocks at all, so `ai-platform` will be the first architecture +consumed as a component. + ### Why LiteLLM Works as a meshStack Platform LiteLLM's native concepts already form a tenancy model, which is what meshStack replicates into: @@ -118,6 +171,37 @@ components live in `modules/ai/`, and each provider contributes only a small mod ![Pluggable seams variant](ai-platform-pluggable.svg) +### The Model Seam as a Contract + +The model seam is a plain map keyed by the model name application teams request, deliberately split so +that only the secrets are marked sensitive: + +```hcl +variable "model_backends" { + description = "OpenAI-compatible model backends registered in the LiteLLM gateway, keyed by model name." + type = map(object({ + litellm_model = string # "openai/neuralmagic/Mistral-7B", "azure/gpt-4o-prod" + api_base = string + extra_params = optional(map(string), {}) # provider quirks, e.g. api_version + })) +} + +variable "model_backend_api_keys" { + description = "API keys for the model backends, keyed by the same model name as model_backends." + type = map(string) + sensitive = true +} +``` + +Splitting the keys out matters in practice: marking one combined structure `sensitive` would collapse +model names, endpoints and versions into `(sensitive value)` in every plan, hiding exactly the +human-readable detail a reviewer needs to check. Keeping the two maps in step is the caller's job, and +the architecture validates that every `model_backends` key has a matching entry. + +This keeps `ai-platform` free of the `stackit` and `azurerm` providers entirely β€” it works because +STACKIT AI Model Serving, Azure OpenAI and self-hosted vLLM are all OpenAI-compatible from LiteLLM's +point of view. "Bring your own model" is then a config value, not a hub PR. + Adding a cloud therefore means adding one small model-access module with the same output shape β€” not changing the architecture. The runtime seam needs no per-cloud work at all: the precedent is [`kubernetes/manifest`](../../modules/kubernetes/manifest), a runtime-agnostic Helm building block @@ -126,6 +210,35 @@ that takes a kubeconfig and declares `supportedPlatforms: kubernetes`. This architecture **consumes** a Kubernetes cluster, it does not provision one β€” which is why [`stackit-kubernetes`](../stackit-kubernetes) stays a separate, companion reference architecture. +### Prerequisite: Cluster Ingress and TLS + +LiteLLM and Langfuse both need a routable HTTPS endpoint with a valid certificate, so the runtime seam +has one requirement beyond "hands out namespaces": an ingress controller and a certificate issuer. That +capability does not exist in the hub yet β€” `modules/kubernetes/` holds only `manifest` and +`service-account` β€” and it is currently **copy-pasted across the foundation repos**: + +| Location | cert-manager | Notes | +|----------|--------------|-------| +| `likvid-cloudfoundation` β€” SKE | v1.20.0 | | +| `internal-cloudfoundation` β€” SKE | v1.20.0 | byte-identical to likvid's SKE copy | +| `trial-cloudfoundation` β€” SKE | v1.20.0 | byte-identical to likvid's SKE copy | +| `likvid-cloudfoundation` β€” AKS | v1.19.4 | plus an Azure load-balancer health-probe annotation | + +All four are `platforms/*/kubernetes/addons/` β€” cert-manager, an HAProxy ingress controller, and a +Let's Encrypt `ClusterIssuer` β€” and the version drift between the SKE and AKS copies is the argument +for consolidating them into a single `modules/kubernetes/ingress`. + +The module is shaped by **capability, not by tool**: one building block delivering "my services get a +public HTTPS URL with a valid certificate". Bundling the issuer with the controller is not arbitrary β€” +the foundations' `ClusterIssuer` hardcodes `ingressClassName = "haproxy"` in its HTTP-01 solver, so the +issuer is not independently useful. + +It also lets the foundations drop a module. Today the `ClusterIssuer` needs its own terragrunt unit +because `kubernetes_manifest` requires the CRD to exist at plan time. Rendering it through a local Helm +chart instead β€” the `chart = path.module` pattern +[`kubernetes/manifest`](../../modules/kubernetes/manifest) already uses β€” removes the plan-time schema +lookup, so `addons/` and `addons/certmanager/` collapse into one. + ## Deployment vs Tenancy The platform components are deployed **once** and are internally multi-tenant. Nothing is deployed per @@ -191,12 +304,16 @@ consequences to design for: 1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing - `ai/litellm-team` β€” none of which are written yet. -2. **Bootstrap ordering β€” resolved for one apply.** Provider configurations *may* reference resource - attributes, but fail when the value is unknown at plan time. So the RA generates the LiteLLM admin - key itself (`random_password`), passes it into the Helm values and derives the endpoint from a known - hostname β€” both knowable up front, leaving `depends_on` sufficient. `stackit-landingzone` already - uses `depends_on` between a platform and a building block instance in one apply. + `ai/litellm-team` β€” plus `modules/kubernetes/ingress` for the TLS/ingress prerequisite. None are + written yet. This architecture also still needs its own `buildingblock/` and + `meshstack_integration.tf` to become orderable. +2. **Bootstrap ordering β€” resolved for one apply.** Two credentials, two mechanisms. The *cluster* + credential is a `STATIC` encrypted input read back through `file()`, so it is known at plan time. + The *LiteLLM admin key* is generated by the architecture itself (`random_password`) and passed into + the Helm values, with the endpoint derived from a known hostname β€” so nothing needs to be read out + of a resource that has not been created yet, and `depends_on` is sufficient. Provider configurations + *may* reference resource attributes; they fail only when the value is unknown at plan time, which + neither of these is. 3. **Provisioning mechanism β€” resolved.** No LiteLLM or Langfuse Terraform provider exists, so `ai/litellm-team` drives both admin APIs with `Mastercard/restapi`, already the established pattern in this repo (13 usages), with `hashicorp/helm` for the chart deploys. No new pattern needed. @@ -216,22 +333,35 @@ consequences to design for: Kubernetes platform. By contrast `meshstack_landingzone.platform_ref` already targets a specific platform by uuid. -This is why step β‘  is RA-owned Terraform rather than a catalog block. Allowing `supported_platforms` -to reference a `meshPlatform` would make the catalog path pluggable and is worth having: +This does **not** block step β‘ , since a `WORKSPACE_LEVEL` block needs no `supported_platforms` at all. +It bites on the tenant-facing blocks and on making cluster choice an order-time decision: - The realistic topology is **two SKE clusters** β€” one hosting the shared AI platform, one hosting - application workloads such as the SKE Starterkit. Without instance-level support, an `ai-platform` + application workloads such as the SKE Starterkit. Without instance-level support, a `TENANT_LEVEL` block offered for type `kubernetes` appears orderable on both. -- It would let the platform engineer answer "which cluster?" as a normal platform reference instead of - threading a kubeconfig or platform uuid through Terraform inputs. -- The current workaround β€” registering each cluster as its own custom platform type β€” inflates the +- It would let the platform engineer answer "which cluster?" as a normal platform reference. The + workaround today is one BBD per cluster with the kubeconfig baked in as a static input, or a + `PLATFORM_OPERATOR_MANUAL_INPUT` kubeconfig field β€” which does support `sensitive`, but turns a + reference into hand-carried credentials. +- The other workaround β€” registering each cluster as its own custom platform type β€” inflates the platform-type list to express what is really an instance selection. -**Sensitive outputs between building blocks.** If meshStack supported encrypted sensitive outputs from -one building block into another, this architecture could be split into two properly dependent blocks β€” -one deploying the gateway, one registering the platform and consuming its endpoint and admin -credential β€” instead of relying on a pre-generated credential to keep everything in one apply. That is -the cleaner decomposition and would remove the plan-time-unknown constraint entirely. +**Sensitive outputs between building blocks.** The gap here is narrower than it first appears, and +worth stating precisely. Sensitive input values *are* supported β€” for `USER_INPUT`, +`PLATFORM_OPERATOR_MANUAL_INPUT` and `STATIC` β€” which is what makes the encrypted-kubeconfig pattern +above work. They are excluded for exactly one assignment type, `BUILDING_BLOCK_OUTPUT`, and outputs +themselves are typed `STRING | CODE | INTEGER | BOOLEAN` with no sensitive variant. + +So composition itself is fine: `dependency_refs` plus `BUILDING_BLOCK_OUTPUT` already works, and +[`ske/forgejo-connector`](../../modules/ske/forgejo-connector) uses both today. Non-secret facts flow +between blocks without trouble β€” ingress class, cluster issuer name, load-balancer IP, hostname β€” so +this architecture routes around the gap by keeping the **secret path** static and encrypted while the +**dependency path** carries only public values. + +What remains blocked is the genuinely self-service case: one block *creating* a cluster and another +consuming its kubeconfig at runtime. Today that credential would have to be a plaintext `CODE` output +visible in meshPanel. Closing this is the prerequisite for the SKE cluster itself becoming an orderable +building block rather than foundation Terraform. ## Tracked: Folding In the SKE Starterkit @@ -257,17 +387,20 @@ several providers (renaming to `OPENAI_BASE_URL` / `OPENAI_API_KEY` would additi client SDKs work with zero configuration), and routing the demo app through the gateway means its traffic shows up in Langfuse and counts against the team's budget β€” which is the point. -**Delivery idea:** expose AI as an opt-in option the same way -[`stackit-landingzone`](../stackit-landingzone) exposes networking β€” a nullable object variable -(`variable "network"`, unset = sandbox only) β€” with the SKE Starterkit as a further option that -requires the AI option to be enabled. +**Delivery:** AI becomes an opt-in option on +[`stackit-landingzone`](../stackit-landingzone), shaped like its existing `variable "network"`, which +reuses this architecture as a component β€” see +[Composition](#composition-architectures-reuse-architectures). The SKE Starterkit is then a further +option that requires the AI option to be enabled. There is no layering conflict here, because the **SKE cluster is itself an offering inside a STACKIT project**: the STACKIT LZ provisions the project, the SKE cluster building block turns it into a Kubernetes platform, and that platform's landing zone hands out the namespaces the AI components and the starterkit need. Note the cluster building block does not exist in the hub yet β€” SKE clusters are provisioned by foundation Terraform today (`platforms/ske/kubernetes/cluster.tf` in the -cloudfoundation repos), so hub-ifying it is a prerequisite. +cloudfoundation repos), so hub-ifying it is a prerequisite. It is also the one place the sensitive-output +gap genuinely bites: an orderable cluster block would need to hand its kubeconfig to the blocks +installing into it. ## Getting Started From 329b0f081573090c7fb4bfa7fec05d88b46bead0 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 14:15:21 +0200 Subject: [PATCH 11/60] docs(ref-arch): layer stackit-landingzone options, settle SKE exposure The ai option builds on a kubernetes option that does not exist yet: network -> kubernetes -> ai. Record that the network -> kubernetes hop already composes, since stackit/network outputs a non-secret network_id that stackit_ske_cluster.network.id consumes via BUILDING_BLOCK_OUTPUT. Separate the two controls that get conflated: access_scope = SNA privatises only the Kubernetes API and is feature-flagged, immutable and mutually exclusive with the ACL extension, while the GA internal-lb annotation is what actually keeps a workload off the internet. Both are available at the pinned provider version. Default to expose = public with TLS so kubernetes/ingress is exercised end to end, noting the sovereignty trade-off, with internal as opt-in. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 88 +++++++++++++++++-- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index ed05b07c..1bd27f5b 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -118,6 +118,80 @@ Note this is new ground: no reference architecture reuses another one yet. `stac integration file contains no `module` blocks at all, so `ai-platform` will be the first architecture consumed as a component. +### Option Layering in `stackit-landingzone` + +AI is not the next option to add β€” it sits on top of a Kubernetes option that does not exist yet. The +options stack: + +| Option | Provisions | Builds on | +|--------|------------|-----------| +| `network` | SNA hub network area, plus a tenant-facing `stackit/network` spoke block | β€” | +| `kubernetes` | A `TENANT_LEVEL` block ordering an SKE cluster into a STACKIT project, registering it as a Kubernetes platform with a namespace landing zone | `network`, for SNA placement | +| `ai` | Reuses `ai-platform`, auto-ordering a dedicated SKE cluster through the `kubernetes` option | `kubernetes` | + +**The `network` β†’ `kubernetes` hop composes today.** +[`stackit/network`](../../modules/stackit/network) already outputs `network_id`, which is exactly what +`stackit_ske_cluster.network.id` consumes β€” and because a network id is not a secret, a +`BUILDING_BLOCK_OUTPUT` input carries it with no missing feature. The hop that *does* need sensitive +outputs is the next one: a cluster block handing its kubeconfig to whatever installs into it. + +The `kubernetes` option needs a new `modules/stackit/ske-cluster` block; SKE clusters live in +foundation Terraform today. + +The AI option then takes a **nullable cluster target**, defaulting to ordering its own: + +```hcl +variable "ai" { + type = object({ + model = string + expose = optional(string, "public") # public | internal + cluster = optional(object({ # null => order a dedicated cluster + platform_identifier = string + landingzone = string + }), null) + }) + default = null +} +``` + +Auto-ordering a **dedicated** cluster (dedicated to the platform team β€” distinct from a *private* +cluster in the networking sense below) is the default because it makes the architecture one click from +an empty STACKIT organization, and because the architecture then creates the cluster it installs into, +which sidesteps the instance-level `supported_platforms` limitation entirely. Pointing at an existing +cluster stays available for cheaper demos. + +### Exposure: Public TLS by Default, Internal Opt-In + +Two independent controls exist, and they are easy to conflate: + +| Control | What it privatises | Status | +|---------|--------------------|--------| +| `network.control_plane.access_scope = "SNA"` | The **Kubernetes API** only | Feature-flagged, **not GA** β€” needs a STACKIT support ticket per org/project | +| `lb.stackit.cloud/internal-lb: "true"` | The **workload's** LoadBalancer address | GA, no flag, no ticket | + +A private control plane does nothing to keep LiteLLM off the internet β€” that is a Service concern. The +internal-LB annotation gives the LoadBalancer an address from the node network instead of a floating +public IP, and it works as a private alternative *because* tenant projects already share the SNA when +the `network` option is enabled. Both are available in the provider version this repo already pins +(`>= 0.88.0`). + +Two further constraints on the private-cluster path: `access_scope` is **immutable**, so it cannot be +flipped on an existing cluster without replacement, and it is **mutually exclusive with the ACL +extension** β€” private control plane, or public control plane with `extensions.acl.allowed_cidrs`, not +both. + +**The default is `expose = "public"`**: HAProxy plus cert-manager and a Let's Encrypt `ClusterIssuer`, +the path the foundations already run. This exercises +[`kubernetes/ingress`](#prerequisite-cluster-ingress-and-tls) end-to-end and means the Langfuse UI +simply works in a browser. The trade-off is deliberate and worth stating plainly: the sovereign gateway +is then reachable from the internet, defended by virtual keys and optionally +`spec.loadBalancerSourceRanges`. Sovereignty here is about *where the data is processed*, not about +network reachability. + +`expose = "internal"` inverts this for customers who need it, at two costs: the Langfuse UI needs VPN +or on-prem connectivity, and HTTP-01 certificate solving does not work against a private address, so +that path needs DNS-01 or an internal CA. + ### Why LiteLLM Works as a meshStack Platform LiteLLM's native concepts already form a tenancy model, which is what meshStack replicates into: @@ -304,9 +378,10 @@ consequences to design for: 1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing - `ai/litellm-team` β€” plus `modules/kubernetes/ingress` for the TLS/ingress prerequisite. None are - written yet. This architecture also still needs its own `buildingblock/` and - `meshstack_integration.tf` to become orderable. + `ai/litellm-team` β€” plus `modules/kubernetes/ingress` for the TLS/ingress prerequisite and + `modules/stackit/ske-cluster` for the `kubernetes` option the AI option builds on. None are written + yet. This architecture also still needs its own `buildingblock/` and `meshstack_integration.tf` to + become orderable. 2. **Bootstrap ordering β€” resolved for one apply.** Two credentials, two mechanisms. The *cluster* credential is a `STATIC` encrypted input read back through `file()`, so it is known at plan time. The *LiteLLM admin key* is generated by the architecture itself (`random_password`) and passed into @@ -398,9 +473,10 @@ project**: the STACKIT LZ provisions the project, the SKE cluster building block Kubernetes platform, and that platform's landing zone hands out the namespaces the AI components and the starterkit need. Note the cluster building block does not exist in the hub yet β€” SKE clusters are provisioned by foundation Terraform today (`platforms/ske/kubernetes/cluster.tf` in the -cloudfoundation repos), so hub-ifying it is a prerequisite. It is also the one place the sensitive-output -gap genuinely bites: an orderable cluster block would need to hand its kubeconfig to the blocks -installing into it. +cloudfoundation repos), so hub-ifying it as `modules/stackit/ske-cluster` behind the `kubernetes` +option is a prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). It is also +the one place the sensitive-output gap genuinely bites: an orderable cluster block would need to hand +its kubeconfig to the blocks installing into it. ## Getting Started From 4b76bd355004290549f30450f2f1b3764dbdef3d Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 14:28:13 +0200 Subject: [PATCH 12/60] docs(ref-arch): add self-registering SKE cluster block as modules/stackit/ske Place the cluster block under the platform it is ordered on, mirroring modules/azure/aks (name: AKS Cluster, supportedPlatforms: [azure]), so modules/ske and modules/aks stay the platforms whose tenants are namespaces. The block is TENANT_LEVEL on STACKIT and registers its own platform and landing zone via an ephemeral meshStack API token, which keeps the kubeconfig inside the run - so the sensitive-output gap is avoided on this hop rather than blocking it. Reframe what the gap actually costs: decomposition freedom, not capability. Also note ske-starterkit already threads platform_ref as a static input, so instance-level targeting is an ergonomics request, not a missing capability. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index 1bd27f5b..1f64322b 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -135,8 +135,44 @@ options stack: `BUILDING_BLOCK_OUTPUT` input carries it with no missing feature. The hop that *does* need sensitive outputs is the next one: a cluster block handing its kubeconfig to whatever installs into it. -The `kubernetes` option needs a new `modules/stackit/ske-cluster` block; SKE clusters live in -foundation Terraform today. +#### The `kubernetes` Option: A Self-Registering Cluster Block + +The option needs a new `modules/stackit/ske` block β€” SKE clusters live in foundation Terraform today. +It goes beside [`stackit/project`](../../modules/stackit/project), +[`stackit/network`](../../modules/stackit/network) and +[`stackit/model-serving`](../../modules/stackit/model-serving) because **the directory follows the +platform the block is ordered on**, not the thing it produces. This mirrors +[`modules/azure/aks`](../../modules/azure/aks) (`name: AKS Cluster`, `supportedPlatforms: [azure]`) +exactly, and keeps `modules/ske/` and `modules/aks/` as what their readmes describe: the platforms +whose tenants are *namespaces*. + +The block is `TENANT_LEVEL` on the STACKIT platform, so its tenant is the STACKIT project, and it +**registers its own platform and landing zone** using an ephemeral meshStack API token: + +```hcl +resource "meshstack_building_block_definition" "ske" { + spec = { + target_type = "TENANT_LEVEL" + supported_platforms = [{ name = "STACKIT" }] + } + version_spec = { + permissions = [/* platform + landing zone admin */] # ephemeral meshStack API key + inputs = { + project_id = { assignment_type = "PLATFORM_TENANT_ID" } + network_id = { + assignment_type = "BUILDING_BLOCK_OUTPUT" + argument = jsonencode("${var.network_bbd_uuid}.network_id") + } + } + } +} +``` + +This is the decisive property: because the block creates the cluster *and* registers it, **the +kubeconfig never crosses a building block boundary** β€” so the sensitive-output gap does not apply to +this hop either. Ordering a cluster yields a ready-to-use Kubernetes platform with a namespace landing +zone in one atomic step. The trade-off accepted here is that a tenant-ordered block holds permission to +create meshStack platforms and landing zones. The AI option then takes a **nullable cluster target**, defaulting to ordering its own: @@ -379,8 +415,8 @@ consequences to design for: 1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing `ai/litellm-team` β€” plus `modules/kubernetes/ingress` for the TLS/ingress prerequisite and - `modules/stackit/ske-cluster` for the `kubernetes` option the AI option builds on. None are written - yet. This architecture also still needs its own `buildingblock/` and `meshstack_integration.tf` to + `modules/stackit/ske` for the `kubernetes` option the AI option builds on. None are written yet. + This architecture also still needs its own `buildingblock/` and `meshstack_integration.tf` to become orderable. 2. **Bootstrap ordering β€” resolved for one apply.** Two credentials, two mechanisms. The *cluster* credential is a `STATIC` encrypted input read back through `file()`, so it is known at plan time. @@ -414,8 +450,12 @@ It bites on the tenant-facing blocks and on making cluster choice an order-time - The realistic topology is **two SKE clusters** β€” one hosting the shared AI platform, one hosting application workloads such as the SKE Starterkit. Without instance-level support, a `TENANT_LEVEL` block offered for type `kubernetes` appears orderable on both. -- It would let the platform engineer answer "which cluster?" as a normal platform reference. The - workaround today is one BBD per cluster with the kubeconfig baked in as a static input, or a +- It would let the platform engineer answer "which cluster?" as a normal platform reference. Note the + capability already exists at the *input* level β€” [`ske/ske-starterkit`](../../modules/ske/ske-starterkit) + threads `platform_ref = { uuid, kind = "meshPlatform" }` in as a static input precisely because "the + meshTenant v4 API references platforms by ref". So this request is about **orderability gating and + ergonomics**, not raw capability: `supported_platforms` cannot express what the input already can. +- The alternatives today are one BBD per cluster with the target baked in as a static input, or a `PLATFORM_OPERATOR_MANUAL_INPUT` kubeconfig field β€” which does support `sensitive`, but turns a reference into hand-carried credentials. - The other workaround β€” registering each cluster as its own custom platform type β€” inflates the @@ -433,10 +473,13 @@ between blocks without trouble β€” ingress class, cluster issuer name, load-bala this architecture routes around the gap by keeping the **secret path** static and encrypted while the **dependency path** carries only public values. -What remains blocked is the genuinely self-service case: one block *creating* a cluster and another -consuming its kubeconfig at runtime. Today that credential would have to be a plaintext `CODE` output -visible in meshPanel. Closing this is the prerequisite for the SKE cluster itself becoming an orderable -building block rather than foundation Terraform. +The obvious remaining case β€” a block *creating* a cluster and another consuming its kubeconfig β€” is +avoided rather than blocked, by making the cluster block +[register its own platform](#the-kubernetes-option-a-self-registering-cluster-block) so the credential +never leaves the run. What the gap still costs is **decomposition freedom**: any two capabilities that +must exchange a secret have to be authored as one block, or have the secret injected statically from +outside. That is a design constraint rather than a wall, but it is why "deploy the gateway" and +"register the platform" cannot be separate, independently versioned blocks. ## Tracked: Folding In the SKE Starterkit @@ -473,10 +516,8 @@ project**: the STACKIT LZ provisions the project, the SKE cluster building block Kubernetes platform, and that platform's landing zone hands out the namespaces the AI components and the starterkit need. Note the cluster building block does not exist in the hub yet β€” SKE clusters are provisioned by foundation Terraform today (`platforms/ske/kubernetes/cluster.tf` in the -cloudfoundation repos), so hub-ifying it as `modules/stackit/ske-cluster` behind the `kubernetes` -option is a prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). It is also -the one place the sensitive-output gap genuinely bites: an orderable cluster block would need to hand -its kubeconfig to the blocks installing into it. +cloudfoundation repos), so hub-ifying it as `modules/stackit/ske` behind the `kubernetes` option is a +prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). ## Getting Started From a46b9bc35d406629657c4bb31e443fc52f971414 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Tue, 11 Aug 2026 20:01:21 +0200 Subject: [PATCH 13/60] docs(ref-arch): simplify language across the AI platform branch Replace metaphors and rare words with plain, literal wording: seam -> layer, blast radius -> shared failure, choke point -> single enforcement point, hop -> step, plumbing -> shared infrastructure, sidesteps and routes around -> avoids, privatises -> makes private, hub-ify -> move into the hub. Split sentences that stacked clauses or buried a long aside, and name the actor where a noun hid it. Trim the diagram headers to the bare title the other .dot files use, and drop the ones that restated their own cluster labels or still said "under discussion". Fix the model-serving scaffold note, which pointed at a reference architecture path that no longer exists. Co-Authored-By: Claude Opus 5 (1M context) --- .../model-serving/buildingblock/README.md | 3 +- reference-architectures/ai-platform/README.md | 311 +++++++++--------- .../ai-platform/ai-platform-oneclick.dot | 3 +- .../ai-platform/ai-platform-pluggable.dot | 7 +- .../ai-platform/ai-platform-pluggable.svg | 4 +- .../ai-platform/ai-platform.dot | 2 +- 6 files changed, 165 insertions(+), 165 deletions(-) diff --git a/modules/stackit/model-serving/buildingblock/README.md b/modules/stackit/model-serving/buildingblock/README.md index 854d170b..d4678a55 100644 --- a/modules/stackit/model-serving/buildingblock/README.md +++ b/modules/stackit/model-serving/buildingblock/README.md @@ -7,8 +7,7 @@ description: Issues a scoped STACKIT AI Model Serving API token so a tenant can # STACKIT AI Model Serving Access Building Block - + This building block issues a STACKIT AI Model Serving API token scoped to a tenant's STACKIT project. diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index 1f64322b..be6c5566 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -4,7 +4,7 @@ description: > An opinionated, one-click AI platform: LiteLLM and Langfuse installed into a Kubernetes namespace landing zone, with LiteLLM registered as a meshStack platform so application teams order governed model access β€” budget, allowed models and tracing included β€” as a self-service item. The runtime and - the model backend are both pluggable; STACKIT is the sovereign reference instantiation. + the model backend are both pluggable; STACKIT is the sovereign reference implementation. cloudProviders: - stackit - azure @@ -17,30 +17,30 @@ buildingBlocks: ## Overview - + -Enterprise AI adoption stalls on three questions a raw model endpoint does not answer: *who may call -which model*, *what did it cost*, and *what exactly was sent and returned*. This reference -architecture answers them with opinionated defaults rather than a toolkit: one order installs the -gateway and observability stack, and a second turns model access into a governed, self-service +Three questions hold back enterprise AI adoption, and a raw model endpoint answers none of them: +*who may call which model*, *what did it cost*, and *what exactly was sent and returned*. This +reference architecture answers them with opinionated defaults rather than a toolkit. One order installs +the gateway and observability stack, and a second turns model access into a governed, self-service catalog item. **Target audience:** - **Platform engineers** who want to offer LLM access as a governed product β€” per-team keys, budgets - and model allow-lists β€” instead of handing out one shared credential. + and model allow-lists β€” instead of giving out one shared credential. - **Application teams** who need a stable, governed OpenAI-compatible endpoint to build on β€” chat interfaces, agents, assistants β€” without operating model infrastructure. ## Architecture -The diagram below shows the **STACKIT instantiation** β€” the sovereign reference. The platform runs on -SKE and is deliberately just two components. Every call -goes through **LiteLLM**, the single choke point where virtual keys, budgets and model allow-lists are -enforced, and which routes inference to **STACKIT AI Model Serving**. **Langfuse** traces every call -for evaluation and usage attribution. Self-hosted **vLLM** is shown muted β€” an optional backend, not -required when the managed sovereign API is used. Tenant applications are muted too: what teams build -on the endpoint is their business, not part of the platform. +The diagram below shows the **STACKIT implementation** β€” the sovereign reference. The platform runs on +SKE and has deliberately only two components. Every call goes through **LiteLLM**, which routes +inference to **STACKIT AI Model Serving**. LiteLLM is the single point where virtual keys, budgets and +model allow-lists are enforced. **Langfuse** traces every call for evaluation and usage attribution. +Self-hosted **vLLM** is shown muted β€” an optional backend, not required when the managed sovereign API +is used. Tenant applications are muted too: what teams build on the endpoint is up to them, not part of +the platform. ![AI Platform reference architecture](ai-platform.svg) @@ -52,24 +52,25 @@ into one platform-team order and one application-team order. ![One-click delivery model](ai-platform-oneclick.svg) **β‘  Platform team, once.** An `ai-platform` building block deploys LiteLLM and Langfuse by Helm into a -tenant namespace obtained from an existing Kubernetes landing zone, pre-wired by convention: LiteLLM -points at Langfuse for tracing, and model backends are registered from the credentials it was given. +tenant namespace obtained from an existing Kubernetes landing zone. Convention already wires them +together: LiteLLM points at Langfuse for tracing, and model backends are registered from the +credentials the block was given. -Step β‘  is itself a **building block ordered in the platform team's own workspace** β€” the reference -architecture *is* a building block, following the shape +Step β‘  is itself a **building block ordered in the platform team's own workspace**, because the +reference architecture *is* a building block. It follows the shape [`stackit-landingzone`](../stackit-landingzone) already uses: a thin `meshstack_integration.tf` -declaring one BBD whose implementation points at this architecture's `buildingblock/`, which then -creates the platform, landing zones and the tenant-facing BBDs. The meshStack instance is the -Terraform runtime; the foundation repos only instantiate the architecture. +declares one BBD whose implementation points at this architecture's `buildingblock/`. That building +block then creates the platform, the landing zones and the tenant-facing BBDs. The meshStack instance +is the Terraform runtime; the foundation repos only instantiate the architecture. -Two schema details make this work today, without waiting on any new meshStack feature: +Two schema details make this work today, without any new meshStack feature: - **`target_type = "WORKSPACE_LEVEL"`** (the default) attaches the block to the platform workspace rather than to a tenant. `supported_platforms` is required *only* for `TENANT_LEVEL`, so this step - needs no platform reference at all β€” which is why the instance-level limitation discussed under + needs no platform reference at all. That is why the instance-level limitation described under [Feature Requests](#meshstack-feature-requests) does not block it. - **The cluster credential arrives as an encrypted static input.** The architecture's Terraform holds - the admin kubeconfig, mints a scoped one, and bakes it into the BBD β€” exactly what + the admin kubeconfig, creates a scoped one, and stores it in the BBD β€” exactly what [`kubernetes/manifest`](../../modules/kubernetes/manifest) does at `meshstack_integration.tf:159`: @@ -86,37 +87,37 @@ Two schema details make this work today, without waiting on any new meshStack fe target cluster, so cluster choice is a Terraform variable rather than an order-time selection. **β‘‘ Application team, per team.** LiteLLM is then registered as a **meshStack platform**, so ordering -model access is a normal self-service action: the landing zone carries the policy (allowed models, +model access is a normal self-service action: the landing zone holds the policy (allowed models, budget tier), and the building block creates the LiteLLM team and virtual key behind it. This step is per-project and needs no cluster targeting, so it is unaffected by the limitation below. ### Composition: Architectures Reuse Architectures `ai-platform` is deliberately generic β€” it names no cloud and depends on neither the `stackit` nor the -`azurerm` provider. Provider-specific architectures **reuse** it and fill its two seams: +`azurerm` provider. Provider-specific architectures **reuse** it and fill its two replaceable layers: ```hcl # reference-architectures/stackit-landingzone/buildingblock/main.tf module "ai_platform" { source = "github.com/meshcloud/meshstack-hub//reference-architectures/ai-platform/buildingblock?ref=${var.hub.git_ref}" - # ... SKE for the runtime seam, STACKIT Model Serving for the model seam + # ... SKE as the runtime layer, STACKIT Model Serving as the model layer } ``` Composition must use the **git URL with `?ref=${var.hub.git_ref}`**, not a relative `../../ai-platform` path. Relative sources appear only in `e2e/` harnesses and for directories *below* a building block's -`repository_path`; a `../` escape out of the building block's own path is not a pattern this repo -relies on. +`repository_path`. A `../` path that leaves the building block's own directory is not a pattern this +repo uses. -In `stackit-landingzone`, AI becomes an opt-in option shaped exactly like its existing +In `stackit-landingzone`, AI becomes an opt-in option with the same shape as its existing `variable "network"` β€” a nullable `object({...})` with `optional()` fields, unset meaning "sandbox -only". Enabling it plugs SKE in as the runtime and STACKIT AI Model Serving in as the model backend, -and the resulting platform is where the SKE Starterkit can then be integrated (see -[Tracked](#tracked-folding-in-the-ske-starterkit)). +only". Enabling it plugs SKE in as the runtime and STACKIT AI Model Serving in as the model backend. +The resulting platform is where the SKE Starterkit can then be integrated (see +[Tracked](#tracked-including-the-ske-starterkit)). -Note this is new ground: no reference architecture reuses another one yet. `stackit-landingzone`'s -integration file contains no `module` blocks at all, so `ai-platform` will be the first architecture -consumed as a component. +This is new: no reference architecture reuses another one yet. `stackit-landingzone`'s integration file +contains no `module` blocks at all, so `ai-platform` will be the first architecture consumed as a +component. ### Option Layering in `stackit-landingzone` @@ -129,18 +130,18 @@ options stack: | `kubernetes` | A `TENANT_LEVEL` block ordering an SKE cluster into a STACKIT project, registering it as a Kubernetes platform with a namespace landing zone | `network`, for SNA placement | | `ai` | Reuses `ai-platform`, auto-ordering a dedicated SKE cluster through the `kubernetes` option | `kubernetes` | -**The `network` β†’ `kubernetes` hop composes today.** +**The `network` β†’ `kubernetes` step works today.** [`stackit/network`](../../modules/stackit/network) already outputs `network_id`, which is exactly what -`stackit_ske_cluster.network.id` consumes β€” and because a network id is not a secret, a -`BUILDING_BLOCK_OUTPUT` input carries it with no missing feature. The hop that *does* need sensitive -outputs is the next one: a cluster block handing its kubeconfig to whatever installs into it. +`stackit_ske_cluster.network.id` consumes. Because a network id is not a secret, a +`BUILDING_BLOCK_OUTPUT` input carries it with no missing feature. The step that *does* need sensitive +outputs is the next one: a cluster block passing its kubeconfig to whatever installs into it. #### The `kubernetes` Option: A Self-Registering Cluster Block The option needs a new `modules/stackit/ske` block β€” SKE clusters live in foundation Terraform today. -It goes beside [`stackit/project`](../../modules/stackit/project), +It belongs beside [`stackit/project`](../../modules/stackit/project), [`stackit/network`](../../modules/stackit/network) and -[`stackit/model-serving`](../../modules/stackit/model-serving) because **the directory follows the +[`stackit/model-serving`](../../modules/stackit/model-serving), because **the directory follows the platform the block is ordered on**, not the thing it produces. This mirrors [`modules/azure/aks`](../../modules/azure/aks) (`name: AKS Cluster`, `supportedPlatforms: [azure]`) exactly, and keeps `modules/ske/` and `modules/aks/` as what their readmes describe: the platforms @@ -168,11 +169,11 @@ resource "meshstack_building_block_definition" "ske" { } ``` -This is the decisive property: because the block creates the cluster *and* registers it, **the +This is the decisive property. Because the block creates the cluster *and* registers it, **the kubeconfig never crosses a building block boundary** β€” so the sensitive-output gap does not apply to -this hop either. Ordering a cluster yields a ready-to-use Kubernetes platform with a namespace landing -zone in one atomic step. The trade-off accepted here is that a tenant-ordered block holds permission to -create meshStack platforms and landing zones. +this step either. Ordering a cluster produces a ready-to-use Kubernetes platform with a namespace +landing zone in a single step. The trade-off accepted here is that a tenant-ordered block has +permission to create meshStack platforms and landing zones. The AI option then takes a **nullable cluster target**, defaulting to ordering its own: @@ -190,47 +191,47 @@ variable "ai" { } ``` -Auto-ordering a **dedicated** cluster (dedicated to the platform team β€” distinct from a *private* -cluster in the networking sense below) is the default because it makes the architecture one click from -an empty STACKIT organization, and because the architecture then creates the cluster it installs into, -which sidesteps the instance-level `supported_platforms` limitation entirely. Pointing at an existing -cluster stays available for cheaper demos. +Auto-ordering a **dedicated** cluster is the default for two reasons. It makes the architecture one +click from an empty STACKIT organization, and the architecture then creates the cluster it installs +into, which avoids the instance-level `supported_platforms` limitation entirely. Pointing at an +existing cluster stays available for cheaper demos. Note that *dedicated* here means dedicated to the +platform team, which is a different thing from a *private* cluster in the networking sense below. ### Exposure: Public TLS by Default, Internal Opt-In -Two independent controls exist, and they are easy to conflate: +Two independent controls exist, and they are easy to confuse: -| Control | What it privatises | Status | -|---------|--------------------|--------| +| Control | What it makes private | Status | +|---------|-----------------------|--------| | `network.control_plane.access_scope = "SNA"` | The **Kubernetes API** only | Feature-flagged, **not GA** β€” needs a STACKIT support ticket per org/project | | `lb.stackit.cloud/internal-lb: "true"` | The **workload's** LoadBalancer address | GA, no flag, no ticket | -A private control plane does nothing to keep LiteLLM off the internet β€” that is a Service concern. The +A private control plane does not keep LiteLLM off the internet β€” that is a Service concern. The internal-LB annotation gives the LoadBalancer an address from the node network instead of a floating -public IP, and it works as a private alternative *because* tenant projects already share the SNA when -the `network` option is enabled. Both are available in the provider version this repo already pins +public IP. It works as a private alternative *because* tenant projects already share the SNA when the +`network` option is enabled. Both controls are available in the provider version this repo already pins (`>= 0.88.0`). -Two further constraints on the private-cluster path: `access_scope` is **immutable**, so it cannot be -flipped on an existing cluster without replacement, and it is **mutually exclusive with the ACL -extension** β€” private control plane, or public control plane with `extensions.acl.allowed_cidrs`, not -both. +Two further constraints apply to the private-cluster path. `access_scope` is **immutable**, so it +cannot be changed on an existing cluster without replacing it, and it is **mutually exclusive with the +ACL extension** β€” private control plane, or public control plane with `extensions.acl.allowed_cidrs`, +not both. **The default is `expose = "public"`**: HAProxy plus cert-manager and a Let's Encrypt `ClusterIssuer`, -the path the foundations already run. This exercises -[`kubernetes/ingress`](#prerequisite-cluster-ingress-and-tls) end-to-end and means the Langfuse UI -simply works in a browser. The trade-off is deliberate and worth stating plainly: the sovereign gateway -is then reachable from the internet, defended by virtual keys and optionally +the path the foundations already run. This uses +[`kubernetes/ingress`](#prerequisite-cluster-ingress-and-tls) end to end and means the Langfuse UI +works in a browser. The trade-off is deliberate and worth stating plainly: the sovereign gateway is +then reachable from the internet, protected by virtual keys and optionally `spec.loadBalancerSourceRanges`. Sovereignty here is about *where the data is processed*, not about network reachability. -`expose = "internal"` inverts this for customers who need it, at two costs: the Langfuse UI needs VPN -or on-prem connectivity, and HTTP-01 certificate solving does not work against a private address, so -that path needs DNS-01 or an internal CA. +`expose = "internal"` reverses this for customers who need it, at two costs. The Langfuse UI then needs +VPN or on-prem connectivity, and HTTP-01 certificate solving does not work against a private address, +so that path needs DNS-01 or an internal CA. ### Why LiteLLM Works as a meshStack Platform -LiteLLM's native concepts already form a tenancy model, which is what meshStack replicates into: +LiteLLM's native concepts already form a tenancy model, which is what meshStack maps onto: | meshStack | LiteLLM | |-----------|---------| @@ -239,7 +240,7 @@ LiteLLM's native concepts already form a tenancy model, which is what meshStack | Tenant credential | Virtual key | | Project roles | Team membership | -This is expressible today: `meshstack_platform` supports `spec.config.custom.platform_type_ref`, and +This is possible today: `meshstack_platform` supports `spec.config.custom.platform_type_ref`, and there is a precedent in this repo β€” [`modules/stackit`](../../modules/stackit) registers STACKIT itself as a **custom** platform type, with the actual tenant provisioning done by the [`stackit/project`](../../modules/stackit/project) building block. A LiteLLM platform would follow @@ -250,40 +251,40 @@ the same shape, with a `litellm/team` building block in place of `stackit/projec The platform is the **governed API**, not an end-user product. A bundled UI such as OpenWebUI was considered and deliberately left out: -- It is an **application, not plumbing**. LiteLLM and Langfuse are what every AI workload needs; - a chat UI is one specific product built *on* them β€” and teams will build their own. +- It is an **application, not shared infrastructure**. LiteLLM and Langfuse are what every AI workload + needs; a chat UI is one specific product built *on* them β€” and teams will build their own. - It brings its own **user, group and per-group model permissions**, a second policy store competing - with LiteLLM. "Who may call which model" must have exactly one home, and that home is the landing - zone plus the virtual key. + with LiteLLM. "Who may call which model" must be defined in exactly one place, and that place is the + landing zone plus the virtual key. - Its built-in RAG stack duplicates the already-dropped RAG layer. -- A shared UI cuts across the tenant boundary the virtual key defines, forcing its user list to be - reconciled against meshStack projects. +- A shared UI spans the tenant boundary the virtual key defines, so someone must reconcile its user + list against meshStack projects. -The counter-argument is real β€” a URL you can chat at beats an API key for demos and for business -users who will never write a client. That is why it stays a candidate *optional* catalog block a team -orders into its own namespace with its own key, making it the first example consumer of the platform -rather than part of it. See open question 5. +The counter-argument is real: for demos, and for business users who will never write a client, a URL +you can chat at is worth more than an API key. That is why it stays a candidate *optional* catalog +block a team orders into its own namespace with its own key, making it the first example consumer of +the platform rather than part of it. See open question 5. -## Pluggability: Two Independent Seams +## Pluggability: Two Replaceable Layers -The demo stack was built so infrastructure and model serving are replaceable. That generalises into -**two orthogonal seams**, and because they are orthogonal this is *one* reference architecture rather +The demo stack was built so infrastructure and model serving are replaceable. That becomes **two +independent layers**, and because they are independent this is *one* reference architecture rather than a STACKIT and an Azure fork. Running on SKE while calling Azure OpenAI, or on AKS while calling STACKIT, are both valid combinations. This is why the architecture is named for the capability rather than a cloud: cloud-agnostic components live in `modules/ai/`, and each provider contributes only a small model-access module. -| Seam | Contract | Chosen by | Implementations | -|------|----------|-----------|-----------------| -| **Runtime** β€” where the components run | A landing zone that hands out Kubernetes namespaces; the blocks declare `supportedPlatforms: kubernetes` and never name a cloud | The landing zone the platform team orders into | STACKIT SKE, Azure AKS, any conformant cluster | -| **Model** β€” where inference happens | An OpenAI-compatible endpoint plus credential, surfaced as a LiteLLM `model_list` entry | LiteLLM routing policy, fed by one model-access block per provider | `stackit/model-serving`, an Azure OpenAI equivalent, self-hosted vLLM | +| Layer | Contract | Chosen by | Implementations | +|-------|----------|-----------|-----------------| +| **Runtime** β€” where the components run | A landing zone that provides Kubernetes namespaces; the blocks declare `supportedPlatforms: kubernetes` and never name a cloud | The landing zone the platform team orders into | STACKIT SKE, Azure AKS, any conformant cluster | +| **Model** β€” where inference happens | An OpenAI-compatible endpoint plus credential, registered as a LiteLLM `model_list` entry | LiteLLM routing policy, supplied by one model-access block per provider | `stackit/model-serving`, an Azure OpenAI equivalent, self-hosted vLLM | -![Pluggable seams variant](ai-platform-pluggable.svg) +![Two replaceable layers](ai-platform-pluggable.svg) -### The Model Seam as a Contract +### The Model Layer as a Contract -The model seam is a plain map keyed by the model name application teams request, deliberately split so +The model layer is a plain map keyed by the model name application teams request, deliberately split so that only the secrets are marked sensitive: ```hcl @@ -303,17 +304,17 @@ variable "model_backend_api_keys" { } ``` -Splitting the keys out matters in practice: marking one combined structure `sensitive` would collapse -model names, endpoints and versions into `(sensitive value)` in every plan, hiding exactly the -human-readable detail a reviewer needs to check. Keeping the two maps in step is the caller's job, and -the architecture validates that every `model_backends` key has a matching entry. +Splitting the keys out matters in practice. Marking one combined structure `sensitive` would hide model +names, endpoints and versions behind `(sensitive value)` in every plan β€” exactly the readable detail a +reviewer needs to check. Keeping the two maps consistent is the caller's job, and the architecture +validates that every `model_backends` key has a matching entry. -This keeps `ai-platform` free of the `stackit` and `azurerm` providers entirely β€” it works because +This keeps `ai-platform` free of the `stackit` and `azurerm` providers entirely. It works because STACKIT AI Model Serving, Azure OpenAI and self-hosted vLLM are all OpenAI-compatible from LiteLLM's point of view. "Bring your own model" is then a config value, not a hub PR. Adding a cloud therefore means adding one small model-access module with the same output shape β€” not -changing the architecture. The runtime seam needs no per-cloud work at all: the precedent is +changing the architecture. The runtime layer needs no per-cloud work at all: the precedent is [`kubernetes/manifest`](../../modules/kubernetes/manifest), a runtime-agnostic Helm building block that takes a kubeconfig and declares `supportedPlatforms: kubernetes`. @@ -322,8 +323,8 @@ This architecture **consumes** a Kubernetes cluster, it does not provision one ### Prerequisite: Cluster Ingress and TLS -LiteLLM and Langfuse both need a routable HTTPS endpoint with a valid certificate, so the runtime seam -has one requirement beyond "hands out namespaces": an ingress controller and a certificate issuer. That +LiteLLM and Langfuse both need a routable HTTPS endpoint with a valid certificate, so the runtime layer +has one requirement beyond providing namespaces: an ingress controller and a certificate issuer. That capability does not exist in the hub yet β€” `modules/kubernetes/` holds only `manifest` and `service-account` β€” and it is currently **copy-pasted across the foundation repos**: @@ -343,11 +344,11 @@ public HTTPS URL with a valid certificate". Bundling the issuer with the control the foundations' `ClusterIssuer` hardcodes `ingressClassName = "haproxy"` in its HTTP-01 solver, so the issuer is not independently useful. -It also lets the foundations drop a module. Today the `ClusterIssuer` needs its own terragrunt unit +It also lets the foundations delete a module. Today the `ClusterIssuer` needs its own terragrunt unit, because `kubernetes_manifest` requires the CRD to exist at plan time. Rendering it through a local Helm -chart instead β€” the `chart = path.module` pattern -[`kubernetes/manifest`](../../modules/kubernetes/manifest) already uses β€” removes the plan-time schema -lookup, so `addons/` and `addons/certmanager/` collapse into one. +chart removes that plan-time schema lookup, so `addons/` and `addons/certmanager/` merge into one. +[`kubernetes/manifest`](../../modules/kubernetes/manifest) already uses this `chart = path.module` +pattern. ## Deployment vs Tenancy @@ -360,15 +361,15 @@ application team: | Langfuse | one, shared | Organization/project + scoped API key | `ai/litellm-team` | | STACKIT AI Model Serving | managed service | Token in the tenant's own STACKIT project | `stackit/model-serving` | -This is forced for LiteLLM β€” a gateway only enforces budgets and allow-lists if everything goes -through one instance β€” and chosen for Langfuse, where per-tenant deployments would be disproportionate -(recent Langfuse versions need ClickHouse and Redis alongside Postgres, so each tenant install would -carry a full data stack). +This is forced for LiteLLM, because a gateway only enforces budgets and allow-lists if everything goes +through one instance. For Langfuse it is a choice: per-tenant deployments would be too expensive, since +recent Langfuse versions need ClickHouse and Redis alongside Postgres, so each tenant install would +need a full data stack. -The trade-off to accept consciously: **isolation rests on Langfuse's project boundary, not on a -Kubernetes or network boundary**, and the shared instances are a common blast radius β€” if Langfuse is -down, no team has tracing. For a sovereignty story this is usually fine, since the data never leaves -the cluster; a tenant with stricter isolation requirements would need its own deployment, which this +One trade-off to accept deliberately: **isolation depends on Langfuse's project boundary, not on a +Kubernetes or network boundary**, and the shared instances fail together β€” if Langfuse is down, no team +has tracing. For sovereignty requirements this is usually acceptable, since the data never leaves the +cluster. A tenant with stricter isolation requirements would need its own deployment, which this architecture does not attempt. ## Token Scope: Per-Tenant @@ -380,29 +381,30 @@ variant is simpler and legitimate for some platform engineering setups β€” made - **Clean provider hierarchy.** The credential lives in the tenant's own STACKIT project, so the STACKIT resource hierarchy keeps reflecting the tenant structure instead of collapsing all AI usage onto one platform project. -- **Cost attribution without new plumbing.** Because spend lands on the tenant's own project, it flows - through whatever STACKIT cost path the platform already uses, rather than needing an AI-specific - path that reconstructs per-team spend from gateway data and pushes it back into meshStack. -- **Blast radius.** One tenant's credential can be revoked or rotated without touching anyone else. +- **Cost attribution with no new machinery.** Because spend appears on the tenant's own project, it uses + whatever STACKIT cost path the platform already has, rather than needing an AI-specific path that + reconstructs per-team spend from gateway data and pushes it back into meshStack. +- **Independent revocation.** One tenant's credential can be revoked or rotated without touching + anyone else. -**Ordering stays one click.** The app team's flow is identical to the shared-token variant β€” the extra -work is inside the building block, which touches two systems: it issues the STACKIT token in the -tenant's project *and* registers the corresponding deployment, team and virtual key in LiteLLM. Two -consequences to design for: +**Ordering stays one click.** The application team's steps are identical to the shared-token variant. +The extra work is inside the building block, which writes to two systems: it issues the STACKIT token +in the tenant's project *and* registers the corresponding deployment, team and virtual key in LiteLLM. +Two consequences to design for: - **Two-system consistency.** A token created but not registered in LiteLLM leaves an orphan. The block needs to be idempotent and to clean up on partial failure. -- **Rotation touches both.** Token TTL expiry must update the gateway too, not just STACKIT. +- **Rotation must update both.** Token TTL expiry must update the gateway too, not just STACKIT. ## Governance and Observability - + | Concern | Where it is enforced | |---------|----------------------| | Which models a team may call | LiteLLM model allow-list, bound to the virtual key | | Spend per team | LiteLLM budget on the virtual key; project budget in meshStack | -| Rate limiting / noisy neighbours | LiteLLM per-key rate limits | +| Rate limiting and shared-capacity contention | LiteLLM per-key rate limits | | Prompt and response audit | Langfuse traces | | Quality regression tracking | Langfuse evaluations | | Cost attribution and chargeback | Langfuse usage β†’ meshStack project cost tags | @@ -410,8 +412,6 @@ consequences to design for: ## Open Questions - - 1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing `ai/litellm-team` β€” plus `modules/kubernetes/ingress` for the TLS/ingress prerequisite and @@ -420,8 +420,8 @@ consequences to design for: become orderable. 2. **Bootstrap ordering β€” resolved for one apply.** Two credentials, two mechanisms. The *cluster* credential is a `STATIC` encrypted input read back through `file()`, so it is known at plan time. - The *LiteLLM admin key* is generated by the architecture itself (`random_password`) and passed into - the Helm values, with the endpoint derived from a known hostname β€” so nothing needs to be read out + The architecture generates the *LiteLLM admin key* itself (`random_password`) and passes it into + the Helm values, and it derives the endpoint from a known hostname. So nothing needs to be read out of a resource that has not been created yet, and `depends_on` is sufficient. Provider configurations *may* reference resource attributes; they fail only when the value is unknown at plan time, which neither of these is. @@ -445,20 +445,21 @@ Kubernetes platform. By contrast `meshstack_landingzone.platform_ref` already ta platform by uuid. This does **not** block step β‘ , since a `WORKSPACE_LEVEL` block needs no `supported_platforms` at all. -It bites on the tenant-facing blocks and on making cluster choice an order-time decision: +It matters for the tenant-facing blocks, and for making cluster choice an order-time decision: - The realistic topology is **two SKE clusters** β€” one hosting the shared AI platform, one hosting application workloads such as the SKE Starterkit. Without instance-level support, a `TENANT_LEVEL` block offered for type `kubernetes` appears orderable on both. - It would let the platform engineer answer "which cluster?" as a normal platform reference. Note the - capability already exists at the *input* level β€” [`ske/ske-starterkit`](../../modules/ske/ske-starterkit) - threads `platform_ref = { uuid, kind = "meshPlatform" }` in as a static input precisely because "the - meshTenant v4 API references platforms by ref". So this request is about **orderability gating and - ergonomics**, not raw capability: `supported_platforms` cannot express what the input already can. -- The alternatives today are one BBD per cluster with the target baked in as a static input, or a - `PLATFORM_OPERATOR_MANUAL_INPUT` kubeconfig field β€” which does support `sensitive`, but turns a - reference into hand-carried credentials. -- The other workaround β€” registering each cluster as its own custom platform type β€” inflates the + capability already exists at the *input* level: [`ske/ske-starterkit`](../../modules/ske/ske-starterkit) + passes `platform_ref = { uuid, kind = "meshPlatform" }` in as a static input, precisely because "the + meshTenant v4 API references platforms by ref". So this request is about **which platforms a block + can be ordered on**, and about usability β€” not about capability. `supported_platforms` cannot express + what the input already can. +- The alternatives today are one BBD per cluster with the target stored as a static input, or a + `PLATFORM_OPERATOR_MANUAL_INPUT` kubeconfig field. That field does support `sensitive`, but it + replaces a reference with credentials the operator must copy in by hand. +- The other workaround β€” registering each cluster as its own custom platform type β€” grows the platform-type list to express what is really an instance selection. **Sensitive outputs between building blocks.** The gap here is narrower than it first appears, and @@ -468,20 +469,20 @@ above work. They are excluded for exactly one assignment type, `BUILDING_BLOCK_O themselves are typed `STRING | CODE | INTEGER | BOOLEAN` with no sensitive variant. So composition itself is fine: `dependency_refs` plus `BUILDING_BLOCK_OUTPUT` already works, and -[`ske/forgejo-connector`](../../modules/ske/forgejo-connector) uses both today. Non-secret facts flow +[`ske/forgejo-connector`](../../modules/ske/forgejo-connector) uses both today. Non-secret values pass between blocks without trouble β€” ingress class, cluster issuer name, load-balancer IP, hostname β€” so -this architecture routes around the gap by keeping the **secret path** static and encrypted while the +this architecture avoids the gap by keeping the **secret path** static and encrypted while the **dependency path** carries only public values. The obvious remaining case β€” a block *creating* a cluster and another consuming its kubeconfig β€” is avoided rather than blocked, by making the cluster block [register its own platform](#the-kubernetes-option-a-self-registering-cluster-block) so the credential never leaves the run. What the gap still costs is **decomposition freedom**: any two capabilities that -must exchange a secret have to be authored as one block, or have the secret injected statically from -outside. That is a design constraint rather than a wall, but it is why "deploy the gateway" and -"register the platform" cannot be separate, independently versioned blocks. +must exchange a secret have to be written as one block, or have the secret injected statically from +outside. That is a design constraint, not a blocker, but it is why "deploy the gateway" and "register +the platform" cannot be separate, independently versioned blocks. -## Tracked: Folding In the SKE Starterkit +## Tracked: Including the SKE Starterkit Not in scope for the first iteration, recorded so it is not lost. @@ -490,8 +491,9 @@ its credential is injected statically β€” foundations supply a STACKIT model-ser their own `ai.tf`. The goal is for the starterkit to consume this reference architecture instead, so its demo app becomes a live example of a governed AI workload. -**The seam already exists and is already the right shape.** The starterkit injects AI config through -`forgejo-connector`'s `additional_kubernetes_secrets` as a `stackit-ai` secret holding three values: +**The integration point already exists and already has the right shape.** The starterkit injects AI +config through `forgejo-connector`'s `additional_kubernetes_secrets` as a `stackit-ai` secret holding +three values: | Variable | Today | With this architecture | |----------|-------|------------------------| @@ -500,10 +502,10 @@ its demo app becomes a live example of a governed AI workload. | `STACKIT_AI_MODEL` | A fixed model name | A model from the landing zone's allow-list | Because LiteLLM is OpenAI-compatible, this needs **no change to the starterkit's interface** β€” only -different values. Two notes: the `STACKIT_AI_*` prefix becomes a misnomer once the gateway fronts -several providers (renaming to `OPENAI_BASE_URL` / `OPENAI_API_KEY` would additionally make most -client SDKs work with zero configuration), and routing the demo app through the gateway means its -traffic shows up in Langfuse and counts against the team's budget β€” which is the point. +different values. Two notes. The `STACKIT_AI_*` prefix becomes a misnomer once the gateway serves +several providers, and renaming it to `OPENAI_BASE_URL` / `OPENAI_API_KEY` would also make most client +SDKs work with no configuration. And routing the demo app through the gateway means its traffic appears +in Langfuse and counts against the team's budget β€” which is the point. **Delivery:** AI becomes an opt-in option on [`stackit-landingzone`](../stackit-landingzone), shaped like its existing `variable "network"`, which @@ -513,11 +515,11 @@ option that requires the AI option to be enabled. There is no layering conflict here, because the **SKE cluster is itself an offering inside a STACKIT project**: the STACKIT LZ provisions the project, the SKE cluster building block turns it into a -Kubernetes platform, and that platform's landing zone hands out the namespaces the AI components and +Kubernetes platform, and that platform's landing zone provides the namespaces the AI components and the starterkit need. Note the cluster building block does not exist in the hub yet β€” SKE clusters are provisioned by foundation Terraform today (`platforms/ske/kubernetes/cluster.tf` in the -cloudfoundation repos), so hub-ifying it as `modules/stackit/ske` behind the `kubernetes` option is a -prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). +cloudfoundation repos), so moving it into the hub as `modules/stackit/ske` behind the `kubernetes` +option is a prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). ## Getting Started @@ -547,15 +549,16 @@ prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). | Review own traces and evaluations in their Langfuse project | ❌ | βœ… | | Build and operate the AI application or assistant | ❌ | βœ… | -## Notes for the Session +## Scratch Notes - Demo components in scope: LiteLLM, Langfuse, STACKIT AI Model Serving. -- OpenWebUI was dropped from the core: it is an application, not platform plumbing, and its built-in - user/group model permissions would be a second policy store competing with LiteLLM. Its RAG half - also duplicates the already-dropped RAGFlow. Kept as a candidate optional block (see open question 5). +- OpenWebUI was dropped from the core: it is an application, not shared platform infrastructure, and + its built-in user/group model permissions would be a second policy store competing with LiteLLM. Its + RAG half also duplicates the already-dropped RAGFlow. Kept as a candidate optional block (see open + question 5). - Original demo ran on Scaleway; only the runtime and model-serving layers need swapping. - Hub modules still missing for the platform components themselves β€” only `stackit/model-serving` is - scaffolded, as a minimal first cut around `stackit_modelserving_token` (the only AI-specific + scaffolded, as a minimal first version around `stackit_modelserving_token` (the only AI-specific resource in the STACKIT provider, v0.88.0). diff --git a/reference-architectures/ai-platform/ai-platform-oneclick.dot b/reference-architectures/ai-platform/ai-platform-oneclick.dot index 4bf94bed..ecef6d3c 100644 --- a/reference-architectures/ai-platform/ai-platform-oneclick.dot +++ b/reference-architectures/ai-platform/ai-platform-oneclick.dot @@ -1,6 +1,5 @@ /* - * Delivery model under discussion: opinionated one-click install, then LiteLLM registered as a - * custom meshStack platform so model access itself becomes a landing-zone-governed order. + * AI Platform delivery model β€” the two orders. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ digraph ai_platform_oneclick { diff --git a/reference-architectures/ai-platform/ai-platform-pluggable.dot b/reference-architectures/ai-platform/ai-platform-pluggable.dot index 1473624c..53dab906 100644 --- a/reference-architectures/ai-platform/ai-platform-pluggable.dot +++ b/reference-architectures/ai-platform/ai-platform-pluggable.dot @@ -1,6 +1,5 @@ /* - * Variant under discussion: two pluggable seams β€” the Kubernetes platform whose landing zone - * provides the namespace, and the model backends LiteLLM routes to. + * AI Platform β€” the two replaceable layers. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ digraph ai_platform_pluggable { @@ -50,7 +49,7 @@ digraph ai_platform_pluggable { } subgraph cluster_runtime { - label=" Runtime seam Β· any meshStack Kubernetes platform" + label=" Runtime layer Β· any meshStack Kubernetes platform" labeljust=l fontname="Helvetica" fontsize=12 fontcolor="#697180" style="rounded" color="#d5dae0" @@ -65,7 +64,7 @@ digraph ai_platform_pluggable { } subgraph cluster_models { - label=" Model seam Β· OpenAI-compatible backends" + label=" Model layer Β· OpenAI-compatible backends" labeljust=l fontname="Helvetica" fontsize=12 fontcolor="#697180" style="rounded" color="#d5dae0" diff --git a/reference-architectures/ai-platform/ai-platform-pluggable.svg b/reference-architectures/ai-platform/ai-platform-pluggable.svg index 1936b2af..6d1d60c3 100644 --- a/reference-architectures/ai-platform/ai-platform-pluggable.svg +++ b/reference-architectures/ai-platform/ai-platform-pluggable.svg @@ -19,12 +19,12 @@ cluster_runtime -  Runtime seam Β· any meshStack Kubernetes platform +  Runtime layer Β· any meshStack Kubernetes platform cluster_models -  Model seam Β· OpenAI-compatible backends +  Model layer Β· OpenAI-compatible backends diff --git a/reference-architectures/ai-platform/ai-platform.dot b/reference-architectures/ai-platform/ai-platform.dot index a444c7cc..627ab1e3 100644 --- a/reference-architectures/ai-platform/ai-platform.dot +++ b/reference-architectures/ai-platform/ai-platform.dot @@ -1,5 +1,5 @@ /* - * AI Platform reference architecture β€” governed, observable sovereign model serving. + * AI Platform reference architecture β€” the STACKIT implementation. * Conventions: .agents/references/diagrams.md β€” render with: task diagrams */ digraph ai_platform { From d7e8839c2b50eac8184709036603b8a2183763d1 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 22:23:19 +0200 Subject: [PATCH 14/60] docs(ref-arch): rewrite the AI platform README as a published architecture Turn the design document into a published reference-architecture README: front matter naming stackit-kubernetes as the runtime provider and the five modules the architecture composes, overview, architecture diagram, how it works, deployment vs tenancy, governance, getting started with a deployment order, and the shared responsibility matrix. Drop the design rationale, the open questions and the meshStack feature requests; they are carried over separately. Delete the pluggable-layers diagram, which duplicated the runtime/model table. Co-Authored-By: Claude Opus 5 (1M context) --- reference-architectures/ai-platform/README.md | 546 +++--------------- .../ai-platform/ai-platform-pluggable.dot | 90 --- .../ai-platform/ai-platform-pluggable.svg | 179 ------ 3 files changed, 71 insertions(+), 744 deletions(-) delete mode 100644 reference-architectures/ai-platform/ai-platform-pluggable.dot delete mode 100644 reference-architectures/ai-platform/ai-platform-pluggable.svg diff --git a/reference-architectures/ai-platform/README.md b/reference-architectures/ai-platform/README.md index be6c5566..c0254584 100644 --- a/reference-architectures/ai-platform/README.md +++ b/reference-architectures/ai-platform/README.md @@ -1,29 +1,37 @@ --- name: AI Platform description: > - An opinionated, one-click AI platform: LiteLLM and Langfuse installed into a Kubernetes namespace - landing zone, with LiteLLM registered as a meshStack platform so application teams order governed - model access β€” budget, allowed models and tracing included β€” as a self-service item. The runtime and - the model backend are both pluggable; STACKIT is the sovereign reference implementation. + An opinionated, one-click AI platform: LiteLLM and Langfuse deployed into a Kubernetes namespace, + with LiteLLM registered as a meshStack platform so application teams order governed model access β€” + budget, allowed models and tracing included β€” as a self-service item. The cluster comes from the + stackit-kubernetes reference architecture; the model layer defaults to STACKIT AI Model Serving. cloudProviders: - stackit - azure buildingBlocks: + - path: ai/litellm + role: Deploys the LiteLLM gateway by Helm and registers it as a meshStack platform with its AI landing zones. + - path: ai/langfuse + role: Deploys Langfuse by Helm for tracing, evaluation and usage attribution, and wires LiteLLM to it. + - path: ai/litellm-team + role: Creates the LiteLLM team and virtual key an application team orders, with the models and budget its landing zone allows. - path: stackit/model-serving - role: Issues the STACKIT AI Model Serving credential that LiteLLM routes sovereign inference through. + role: Issues the STACKIT AI Model Serving token that LiteLLM routes sovereign inference through. + - path: ai/azure-openai + role: Registers an Azure OpenAI deployment as a model backend, the Azure counterpart to stackit/model-serving. --- # AI Platform ## Overview - - Three questions hold back enterprise AI adoption, and a raw model endpoint answers none of them: *who may call which model*, *what did it cost*, and *what exactly was sent and returned*. This -reference architecture answers them with opinionated defaults rather than a toolkit. One order installs -the gateway and observability stack, and a second turns model access into a governed, self-service -catalog item. +reference architecture answers them with opinionated defaults rather than a toolkit: one order +installs the gateway and the observability stack, a second turns model access into a catalog item. + +The architecture consumes a Kubernetes cluster and does not provision one. The cluster, its ingress +with TLS and the namespace landing zone come from [`stackit-kubernetes`](../stackit-kubernetes). **Target audience:** @@ -32,206 +40,35 @@ catalog item. - **Application teams** who need a stable, governed OpenAI-compatible endpoint to build on β€” chat interfaces, agents, assistants β€” without operating model infrastructure. -## Architecture +## Architecture Diagram -The diagram below shows the **STACKIT implementation** β€” the sovereign reference. The platform runs on -SKE and has deliberately only two components. Every call goes through **LiteLLM**, which routes -inference to **STACKIT AI Model Serving**. LiteLLM is the single point where virtual keys, budgets and -model allow-lists are enforced. **Langfuse** traces every call for evaluation and usage attribution. -Self-hosted **vLLM** is shown muted β€” an optional backend, not required when the managed sovereign API -is used. Tenant applications are muted too: what teams build on the endpoint is up to them, not part of -the platform. +The diagram shows the STACKIT implementation, the sovereign reference. Every call goes through +**LiteLLM**, which enforces virtual keys, budgets and model allow-lists and routes inference to +**STACKIT AI Model Serving**, while **Langfuse** traces every call. The SKE platform and its landing +zone come from [`stackit-kubernetes`](../stackit-kubernetes); the greyed-out nodes are optional. ![AI Platform reference architecture](ai-platform.svg) -## Delivery Model: Two Orders +## How It Works -The architecture is deliberately opinionated so it is ready to go rather than assembled. It splits -into one platform-team order and one application-team order. +The architecture is opinionated so it is ready to go rather than assembled. It splits into one +platform-team order and one application-team order on the cluster `stackit-kubernetes` provides. ![One-click delivery model](ai-platform-oneclick.svg) -**β‘  Platform team, once.** An `ai-platform` building block deploys LiteLLM and Langfuse by Helm into a -tenant namespace obtained from an existing Kubernetes landing zone. Convention already wires them -together: LiteLLM points at Langfuse for tracing, and model backends are registered from the -credentials the block was given. - -Step β‘  is itself a **building block ordered in the platform team's own workspace**, because the -reference architecture *is* a building block. It follows the shape -[`stackit-landingzone`](../stackit-landingzone) already uses: a thin `meshstack_integration.tf` -declares one BBD whose implementation points at this architecture's `buildingblock/`. That building -block then creates the platform, the landing zones and the tenant-facing BBDs. The meshStack instance -is the Terraform runtime; the foundation repos only instantiate the architecture. - -Two schema details make this work today, without any new meshStack feature: - -- **`target_type = "WORKSPACE_LEVEL"`** (the default) attaches the block to the platform workspace - rather than to a tenant. `supported_platforms` is required *only* for `TENANT_LEVEL`, so this step - needs no platform reference at all. That is why the instance-level limitation described under - [Feature Requests](#meshstack-feature-requests) does not block it. -- **The cluster credential arrives as an encrypted static input.** The architecture's Terraform holds - the admin kubeconfig, creates a scoped one, and stores it in the BBD β€” exactly what - [`kubernetes/manifest`](../../modules/kubernetes/manifest) does at - `meshstack_integration.tf:159`: - - ```hcl - assignment_type = "STATIC" - sensitive = { argument = { - secret_value = "data:application/yaml;base64,${base64encode(module.backplane.kubeconfig)}" - }} - ``` - - The block then applies in **one pass**, because its provider reads `file("kubeconfig.yaml")` β€” a - value known at plan time. There is no unknown-after-apply problem to solve and no terragrunt-style - two-unit split. The consequence to accept: because the credential is static, there is one BBD per - target cluster, so cluster choice is a Terraform variable rather than an order-time selection. - -**β‘‘ Application team, per team.** LiteLLM is then registered as a **meshStack platform**, so ordering -model access is a normal self-service action: the landing zone holds the policy (allowed models, -budget tier), and the building block creates the LiteLLM team and virtual key behind it. This step is -per-project and needs no cluster targeting, so it is unaffected by the limitation below. - -### Composition: Architectures Reuse Architectures - -`ai-platform` is deliberately generic β€” it names no cloud and depends on neither the `stackit` nor the -`azurerm` provider. Provider-specific architectures **reuse** it and fill its two replaceable layers: - -```hcl -# reference-architectures/stackit-landingzone/buildingblock/main.tf -module "ai_platform" { - source = "github.com/meshcloud/meshstack-hub//reference-architectures/ai-platform/buildingblock?ref=${var.hub.git_ref}" - # ... SKE as the runtime layer, STACKIT Model Serving as the model layer -} -``` - -Composition must use the **git URL with `?ref=${var.hub.git_ref}`**, not a relative `../../ai-platform` -path. Relative sources appear only in `e2e/` harnesses and for directories *below* a building block's -`repository_path`. A `../` path that leaves the building block's own directory is not a pattern this -repo uses. - -In `stackit-landingzone`, AI becomes an opt-in option with the same shape as its existing -`variable "network"` β€” a nullable `object({...})` with `optional()` fields, unset meaning "sandbox -only". Enabling it plugs SKE in as the runtime and STACKIT AI Model Serving in as the model backend. -The resulting platform is where the SKE Starterkit can then be integrated (see -[Tracked](#tracked-including-the-ske-starterkit)). - -This is new: no reference architecture reuses another one yet. `stackit-landingzone`'s integration file -contains no `module` blocks at all, so `ai-platform` will be the first architecture consumed as a -component. - -### Option Layering in `stackit-landingzone` - -AI is not the next option to add β€” it sits on top of a Kubernetes option that does not exist yet. The -options stack: - -| Option | Provisions | Builds on | -|--------|------------|-----------| -| `network` | SNA hub network area, plus a tenant-facing `stackit/network` spoke block | β€” | -| `kubernetes` | A `TENANT_LEVEL` block ordering an SKE cluster into a STACKIT project, registering it as a Kubernetes platform with a namespace landing zone | `network`, for SNA placement | -| `ai` | Reuses `ai-platform`, auto-ordering a dedicated SKE cluster through the `kubernetes` option | `kubernetes` | - -**The `network` β†’ `kubernetes` step works today.** -[`stackit/network`](../../modules/stackit/network) already outputs `network_id`, which is exactly what -`stackit_ske_cluster.network.id` consumes. Because a network id is not a secret, a -`BUILDING_BLOCK_OUTPUT` input carries it with no missing feature. The step that *does* need sensitive -outputs is the next one: a cluster block passing its kubeconfig to whatever installs into it. - -#### The `kubernetes` Option: A Self-Registering Cluster Block - -The option needs a new `modules/stackit/ske` block β€” SKE clusters live in foundation Terraform today. -It belongs beside [`stackit/project`](../../modules/stackit/project), -[`stackit/network`](../../modules/stackit/network) and -[`stackit/model-serving`](../../modules/stackit/model-serving), because **the directory follows the -platform the block is ordered on**, not the thing it produces. This mirrors -[`modules/azure/aks`](../../modules/azure/aks) (`name: AKS Cluster`, `supportedPlatforms: [azure]`) -exactly, and keeps `modules/ske/` and `modules/aks/` as what their readmes describe: the platforms -whose tenants are *namespaces*. - -The block is `TENANT_LEVEL` on the STACKIT platform, so its tenant is the STACKIT project, and it -**registers its own platform and landing zone** using an ephemeral meshStack API token: - -```hcl -resource "meshstack_building_block_definition" "ske" { - spec = { - target_type = "TENANT_LEVEL" - supported_platforms = [{ name = "STACKIT" }] - } - version_spec = { - permissions = [/* platform + landing zone admin */] # ephemeral meshStack API key - inputs = { - project_id = { assignment_type = "PLATFORM_TENANT_ID" } - network_id = { - assignment_type = "BUILDING_BLOCK_OUTPUT" - argument = jsonencode("${var.network_bbd_uuid}.network_id") - } - } - } -} -``` - -This is the decisive property. Because the block creates the cluster *and* registers it, **the -kubeconfig never crosses a building block boundary** β€” so the sensitive-output gap does not apply to -this step either. Ordering a cluster produces a ready-to-use Kubernetes platform with a namespace -landing zone in a single step. The trade-off accepted here is that a tenant-ordered block has -permission to create meshStack platforms and landing zones. - -The AI option then takes a **nullable cluster target**, defaulting to ordering its own: - -```hcl -variable "ai" { - type = object({ - model = string - expose = optional(string, "public") # public | internal - cluster = optional(object({ # null => order a dedicated cluster - platform_identifier = string - landingzone = string - }), null) - }) - default = null -} -``` - -Auto-ordering a **dedicated** cluster is the default for two reasons. It makes the architecture one -click from an empty STACKIT organization, and the architecture then creates the cluster it installs -into, which avoids the instance-level `supported_platforms` limitation entirely. Pointing at an -existing cluster stays available for cheaper demos. Note that *dedicated* here means dedicated to the -platform team, which is a different thing from a *private* cluster in the networking sense below. - -### Exposure: Public TLS by Default, Internal Opt-In - -Two independent controls exist, and they are easy to confuse: - -| Control | What it makes private | Status | -|---------|-----------------------|--------| -| `network.control_plane.access_scope = "SNA"` | The **Kubernetes API** only | Feature-flagged, **not GA** β€” needs a STACKIT support ticket per org/project | -| `lb.stackit.cloud/internal-lb: "true"` | The **workload's** LoadBalancer address | GA, no flag, no ticket | - -A private control plane does not keep LiteLLM off the internet β€” that is a Service concern. The -internal-LB annotation gives the LoadBalancer an address from the node network instead of a floating -public IP. It works as a private alternative *because* tenant projects already share the SNA when the -`network` option is enabled. Both controls are available in the provider version this repo already pins -(`>= 0.88.0`). - -Two further constraints apply to the private-cluster path. `access_scope` is **immutable**, so it -cannot be changed on an existing cluster without replacing it, and it is **mutually exclusive with the -ACL extension** β€” private control plane, or public control plane with `extensions.acl.allowed_cidrs`, -not both. - -**The default is `expose = "public"`**: HAProxy plus cert-manager and a Let's Encrypt `ClusterIssuer`, -the path the foundations already run. This uses -[`kubernetes/ingress`](#prerequisite-cluster-ingress-and-tls) end to end and means the Langfuse UI -works in a browser. The trade-off is deliberate and worth stating plainly: the sovereign gateway is -then reachable from the internet, protected by virtual keys and optionally -`spec.loadBalancerSourceRanges`. Sovereignty here is about *where the data is processed*, not about -network reachability. - -`expose = "internal"` reverses this for customers who need it, at two costs. The Langfuse UI then needs -VPN or on-prem connectivity, and HTTP-01 certificate solving does not work against a private address, -so that path needs DNS-01 or an internal CA. - -### Why LiteLLM Works as a meshStack Platform - -LiteLLM's native concepts already form a tenancy model, which is what meshStack maps onto: +### 1. The platform team installs LiteLLM and Langfuse + +The platform team orders the **AI Platform** building block once, in its own workspace. It deploys +both components by Helm into a namespace of the Kubernetes landing zone and wires them together: +LiteLLM points at Langfuse for tracing, and the model backends come from the credentials it was +given. The kubeconfig is an encrypted static input, so one building block definition targets one +cluster. + +### 2. LiteLLM is registered as a meshStack platform + +The same order registers LiteLLM as a meshStack platform of a custom platform type, because its +native concepts already form a tenancy model that meshStack maps onto. The landing zone carries the +policy, so "small budget, open models" and "approved models only" are ordinary landing zones. | meshStack | LiteLLM | |-----------|---------| @@ -240,120 +77,30 @@ LiteLLM's native concepts already form a tenancy model, which is what meshStack | Tenant credential | Virtual key | | Project roles | Team membership | -This is possible today: `meshstack_platform` supports `spec.config.custom.platform_type_ref`, and -there is a precedent in this repo β€” [`modules/stackit`](../../modules/stackit) registers STACKIT -itself as a **custom** platform type, with the actual tenant provisioning done by the -[`stackit/project`](../../modules/stackit/project) building block. A LiteLLM platform would follow -the same shape, with a `litellm/team` building block in place of `stackit/project`. - -### Why No Chat UI in the Core - -The platform is the **governed API**, not an end-user product. A bundled UI such as OpenWebUI was -considered and deliberately left out: - -- It is an **application, not shared infrastructure**. LiteLLM and Langfuse are what every AI workload - needs; a chat UI is one specific product built *on* them β€” and teams will build their own. -- It brings its own **user, group and per-group model permissions**, a second policy store competing - with LiteLLM. "Who may call which model" must be defined in exactly one place, and that place is the - landing zone plus the virtual key. -- Its built-in RAG stack duplicates the already-dropped RAG layer. -- A shared UI spans the tenant boundary the virtual key defines, so someone must reconcile its user - list against meshStack projects. +### 3. Application teams order model access -The counter-argument is real: for demos, and for business users who will never write a client, a URL -you can chat at is worth more than an API key. That is why it stays a candidate *optional* catalog -block a team orders into its own namespace with its own key, making it the first example consumer of -the platform rather than part of it. See open question 5. +Ordering model access is a normal self-service action per project. The `ai/litellm-team` building +block issues a STACKIT AI Model Serving token in the tenant's **own** STACKIT project β€” one model +credential per tenant, not a shared one β€” and registers the matching deployment, team and virtual +key in LiteLLM. The team receives an endpoint, a key and a Langfuse project. -## Pluggability: Two Replaceable Layers +### 4. Two replaceable layers -The demo stack was built so infrastructure and model serving are replaceable. That becomes **two -independent layers**, and because they are independent this is *one* reference architecture rather -than a STACKIT and an Azure fork. Running on SKE while calling Azure OpenAI, or on AKS while calling -STACKIT, are both valid combinations. - -This is why the architecture is named for the capability rather than a cloud: cloud-agnostic -components live in `modules/ai/`, and each provider contributes only a small model-access module. +The runtime and the model backend are independent, which is why this is one reference architecture +rather than a STACKIT fork and an Azure fork: SKE calling Azure OpenAI is as valid as AKS calling +STACKIT. | Layer | Contract | Chosen by | Implementations | |-------|----------|-----------|-----------------| -| **Runtime** β€” where the components run | A landing zone that provides Kubernetes namespaces; the blocks declare `supportedPlatforms: kubernetes` and never name a cloud | The landing zone the platform team orders into | STACKIT SKE, Azure AKS, any conformant cluster | -| **Model** β€” where inference happens | An OpenAI-compatible endpoint plus credential, registered as a LiteLLM `model_list` entry | LiteLLM routing policy, supplied by one model-access block per provider | `stackit/model-serving`, an Azure OpenAI equivalent, self-hosted vLLM | - -![Two replaceable layers](ai-platform-pluggable.svg) - -### The Model Layer as a Contract - -The model layer is a plain map keyed by the model name application teams request, deliberately split so -that only the secrets are marked sensitive: - -```hcl -variable "model_backends" { - description = "OpenAI-compatible model backends registered in the LiteLLM gateway, keyed by model name." - type = map(object({ - litellm_model = string # "openai/neuralmagic/Mistral-7B", "azure/gpt-4o-prod" - api_base = string - extra_params = optional(map(string), {}) # provider quirks, e.g. api_version - })) -} - -variable "model_backend_api_keys" { - description = "API keys for the model backends, keyed by the same model name as model_backends." - type = map(string) - sensitive = true -} -``` - -Splitting the keys out matters in practice. Marking one combined structure `sensitive` would hide model -names, endpoints and versions behind `(sensitive value)` in every plan β€” exactly the readable detail a -reviewer needs to check. Keeping the two maps consistent is the caller's job, and the architecture -validates that every `model_backends` key has a matching entry. - -This keeps `ai-platform` free of the `stackit` and `azurerm` providers entirely. It works because -STACKIT AI Model Serving, Azure OpenAI and self-hosted vLLM are all OpenAI-compatible from LiteLLM's -point of view. "Bring your own model" is then a config value, not a hub PR. - -Adding a cloud therefore means adding one small model-access module with the same output shape β€” not -changing the architecture. The runtime layer needs no per-cloud work at all: the precedent is -[`kubernetes/manifest`](../../modules/kubernetes/manifest), a runtime-agnostic Helm building block -that takes a kubeconfig and declares `supportedPlatforms: kubernetes`. - -This architecture **consumes** a Kubernetes cluster, it does not provision one β€” which is why -[`stackit-kubernetes`](../stackit-kubernetes) stays a separate, companion reference architecture. - -### Prerequisite: Cluster Ingress and TLS - -LiteLLM and Langfuse both need a routable HTTPS endpoint with a valid certificate, so the runtime layer -has one requirement beyond providing namespaces: an ingress controller and a certificate issuer. That -capability does not exist in the hub yet β€” `modules/kubernetes/` holds only `manifest` and -`service-account` β€” and it is currently **copy-pasted across the foundation repos**: - -| Location | cert-manager | Notes | -|----------|--------------|-------| -| `likvid-cloudfoundation` β€” SKE | v1.20.0 | | -| `internal-cloudfoundation` β€” SKE | v1.20.0 | byte-identical to likvid's SKE copy | -| `trial-cloudfoundation` β€” SKE | v1.20.0 | byte-identical to likvid's SKE copy | -| `likvid-cloudfoundation` β€” AKS | v1.19.4 | plus an Azure load-balancer health-probe annotation | - -All four are `platforms/*/kubernetes/addons/` β€” cert-manager, an HAProxy ingress controller, and a -Let's Encrypt `ClusterIssuer` β€” and the version drift between the SKE and AKS copies is the argument -for consolidating them into a single `modules/kubernetes/ingress`. - -The module is shaped by **capability, not by tool**: one building block delivering "my services get a -public HTTPS URL with a valid certificate". Bundling the issuer with the controller is not arbitrary β€” -the foundations' `ClusterIssuer` hardcodes `ingressClassName = "haproxy"` in its HTTP-01 solver, so the -issuer is not independently useful. - -It also lets the foundations delete a module. Today the `ClusterIssuer` needs its own terragrunt unit, -because `kubernetes_manifest` requires the CRD to exist at plan time. Rendering it through a local Helm -chart removes that plan-time schema lookup, so `addons/` and `addons/certmanager/` merge into one. -[`kubernetes/manifest`](../../modules/kubernetes/manifest) already uses this `chart = path.module` -pattern. +| **Runtime** β€” where the components run | A landing zone that provides Kubernetes namespaces; the blocks declare `supportedPlatforms: kubernetes` and never name a cloud | The landing zone the platform team orders into | STACKIT SKE via `stackit-kubernetes`, Azure AKS, any conformant cluster | +| **Model** β€” where inference happens | An OpenAI-compatible endpoint plus credential, registered as a LiteLLM `model_list` entry | LiteLLM routing policy, supplied by one model-access block per provider | `stackit/model-serving`, `ai/azure-openai`, self-hosted vLLM | -## Deployment vs Tenancy +Adding a cloud means adding one small model-access module with the same output shape, not changing +the architecture. The runtime layer needs no per-cloud work at all: the precedent is +[`kubernetes/manifest`](../../modules/kubernetes/manifest), a Helm building block that takes a +kubeconfig and names no cloud. -The platform components are deployed **once** and are internally multi-tenant. Nothing is deployed per -application team: +## Deployment vs Tenancy | Component | Deployments | Tenancy unit | Provisioned by | |-----------|-------------|--------------|----------------| @@ -361,45 +108,11 @@ application team: | Langfuse | one, shared | Organization/project + scoped API key | `ai/litellm-team` | | STACKIT AI Model Serving | managed service | Token in the tenant's own STACKIT project | `stackit/model-serving` | -This is forced for LiteLLM, because a gateway only enforces budgets and allow-lists if everything goes -through one instance. For Langfuse it is a choice: per-tenant deployments would be too expensive, since -recent Langfuse versions need ClickHouse and Redis alongside Postgres, so each tenant install would -need a full data stack. - -One trade-off to accept deliberately: **isolation depends on Langfuse's project boundary, not on a -Kubernetes or network boundary**, and the shared instances fail together β€” if Langfuse is down, no team -has tracing. For sovereignty requirements this is usually acceptable, since the data never leaves the -cluster. A tenant with stricter isolation requirements would need its own deployment, which this -architecture does not attempt. - -## Token Scope: Per-Tenant - -Each tenant gets **its own STACKIT Model Serving token**, issued into its own STACKIT project, rather -than LiteLLM holding one shared platform credential. This is an opinionated call β€” the shared-token -variant is simpler and legitimate for some platform engineering setups β€” made for three reasons: - -- **Clean provider hierarchy.** The credential lives in the tenant's own STACKIT project, so the - STACKIT resource hierarchy keeps reflecting the tenant structure instead of collapsing all AI usage - onto one platform project. -- **Cost attribution with no new machinery.** Because spend appears on the tenant's own project, it uses - whatever STACKIT cost path the platform already has, rather than needing an AI-specific path that - reconstructs per-team spend from gateway data and pushes it back into meshStack. -- **Independent revocation.** One tenant's credential can be revoked or rotated without touching - anyone else. - -**Ordering stays one click.** The application team's steps are identical to the shared-token variant. -The extra work is inside the building block, which writes to two systems: it issues the STACKIT token -in the tenant's project *and* registers the corresponding deployment, team and virtual key in LiteLLM. -Two consequences to design for: - -- **Two-system consistency.** A token created but not registered in LiteLLM leaves an orphan. The - block needs to be idempotent and to clean up on partial failure. -- **Rotation must update both.** Token TTL expiry must update the gateway too, not just STACKIT. +The platform components are deployed once and are internally multi-tenant; nothing is deployed per +application team. Isolation rests on LiteLLM's team boundary and Langfuse's project boundary. ## Governance and Observability - - | Concern | Where it is enforced | |---------|----------------------| | Which models a team may call | LiteLLM model allow-list, bound to the virtual key | @@ -410,130 +123,27 @@ Two consequences to design for: | Cost attribution and chargeback | Langfuse usage β†’ meshStack project cost tags | | Credential rotation | Building block re-order / token TTL | -## Open Questions - -1. **Module scaffolding.** Only `stackit/model-serving` exists. The cloud-agnostic components belong - in a new `modules/ai/` namespace β€” `ai/litellm`, `ai/langfuse` and the tenant-facing - `ai/litellm-team` β€” plus `modules/kubernetes/ingress` for the TLS/ingress prerequisite and - `modules/stackit/ske` for the `kubernetes` option the AI option builds on. None are written yet. - This architecture also still needs its own `buildingblock/` and `meshstack_integration.tf` to - become orderable. -2. **Bootstrap ordering β€” resolved for one apply.** Two credentials, two mechanisms. The *cluster* - credential is a `STATIC` encrypted input read back through `file()`, so it is known at plan time. - The architecture generates the *LiteLLM admin key* itself (`random_password`) and passes it into - the Helm values, and it derives the endpoint from a known hostname. So nothing needs to be read out - of a resource that has not been created yet, and `depends_on` is sufficient. Provider configurations - *may* reference resource attributes; they fail only when the value is unknown at plan time, which - neither of these is. -3. **Provisioning mechanism β€” resolved.** No LiteLLM or Langfuse Terraform provider exists, so - `ai/litellm-team` drives both admin APIs with `Mastercard/restapi`, already the established pattern - in this repo (13 usages), with `hashicorp/helm` for the chart deploys. No new pattern needed. -4. **Metering.** The custom platform type accepts metering configuration; whether LiteLLM spend can - feed meshStack chargeback is unresolved. -5. **Optional chat UI.** A ready-made UI such as OpenWebUI is valuable for demos and for business - users who will not build their own client. Should it ship as an optional catalog block teams order - into their own namespace with their own virtual key? -6. **Dropped from the demo.** FlowiseAI (agent/workflow builder) and RAGFlow (RAG and data layer) were - left out to keep the focus on serving, observability and governance. Follow-up architecture? - -## meshStack Feature Requests - -**Instance-level `supported_platforms` on building block definitions.** Today -`meshstack_building_block_definition.supported_platforms.kind` is documented as *"Always -`meshPlatformType` for now"* β€” a block can declare it supports `kubernetes`, but not *which* -Kubernetes platform. By contrast `meshstack_landingzone.platform_ref` already targets a specific -platform by uuid. - -This does **not** block step β‘ , since a `WORKSPACE_LEVEL` block needs no `supported_platforms` at all. -It matters for the tenant-facing blocks, and for making cluster choice an order-time decision: - -- The realistic topology is **two SKE clusters** β€” one hosting the shared AI platform, one hosting - application workloads such as the SKE Starterkit. Without instance-level support, a `TENANT_LEVEL` - block offered for type `kubernetes` appears orderable on both. -- It would let the platform engineer answer "which cluster?" as a normal platform reference. Note the - capability already exists at the *input* level: [`ske/ske-starterkit`](../../modules/ske/ske-starterkit) - passes `platform_ref = { uuid, kind = "meshPlatform" }` in as a static input, precisely because "the - meshTenant v4 API references platforms by ref". So this request is about **which platforms a block - can be ordered on**, and about usability β€” not about capability. `supported_platforms` cannot express - what the input already can. -- The alternatives today are one BBD per cluster with the target stored as a static input, or a - `PLATFORM_OPERATOR_MANUAL_INPUT` kubeconfig field. That field does support `sensitive`, but it - replaces a reference with credentials the operator must copy in by hand. -- The other workaround β€” registering each cluster as its own custom platform type β€” grows the - platform-type list to express what is really an instance selection. - -**Sensitive outputs between building blocks.** The gap here is narrower than it first appears, and -worth stating precisely. Sensitive input values *are* supported β€” for `USER_INPUT`, -`PLATFORM_OPERATOR_MANUAL_INPUT` and `STATIC` β€” which is what makes the encrypted-kubeconfig pattern -above work. They are excluded for exactly one assignment type, `BUILDING_BLOCK_OUTPUT`, and outputs -themselves are typed `STRING | CODE | INTEGER | BOOLEAN` with no sensitive variant. - -So composition itself is fine: `dependency_refs` plus `BUILDING_BLOCK_OUTPUT` already works, and -[`ske/forgejo-connector`](../../modules/ske/forgejo-connector) uses both today. Non-secret values pass -between blocks without trouble β€” ingress class, cluster issuer name, load-balancer IP, hostname β€” so -this architecture avoids the gap by keeping the **secret path** static and encrypted while the -**dependency path** carries only public values. - -The obvious remaining case β€” a block *creating* a cluster and another consuming its kubeconfig β€” is -avoided rather than blocked, by making the cluster block -[register its own platform](#the-kubernetes-option-a-self-registering-cluster-block) so the credential -never leaves the run. What the gap still costs is **decomposition freedom**: any two capabilities that -must exchange a secret have to be written as one block, or have the secret injected statically from -outside. That is a design constraint, not a blocker, but it is why "deploy the gateway" and "register -the platform" cannot be separate, independently versioned blocks. - -## Tracked: Including the SKE Starterkit - -Not in scope for the first iteration, recorded so it is not lost. - -The [`ske/ske-starterkit`](../../modules/ske/ske-starterkit) demo app already calls an AI model, but -its credential is injected statically β€” foundations supply a STACKIT model-serving token through -their own `ai.tf`. The goal is for the starterkit to consume this reference architecture instead, so -its demo app becomes a live example of a governed AI workload. - -**The integration point already exists and already has the right shape.** The starterkit injects AI -config through `forgejo-connector`'s `additional_kubernetes_secrets` as a `stackit-ai` secret holding -three values: - -| Variable | Today | With this architecture | -|----------|-------|------------------------| -| `STACKIT_AI_BASE_URL` | STACKIT Model Serving endpoint | The LiteLLM gateway URL | -| `STACKIT_AI_API_KEY` | A statically provisioned token | The team's LiteLLM virtual key | -| `STACKIT_AI_MODEL` | A fixed model name | A model from the landing zone's allow-list | - -Because LiteLLM is OpenAI-compatible, this needs **no change to the starterkit's interface** β€” only -different values. Two notes. The `STACKIT_AI_*` prefix becomes a misnomer once the gateway serves -several providers, and renaming it to `OPENAI_BASE_URL` / `OPENAI_API_KEY` would also make most client -SDKs work with no configuration. And routing the demo app through the gateway means its traffic appears -in Langfuse and counts against the team's budget β€” which is the point. - -**Delivery:** AI becomes an opt-in option on -[`stackit-landingzone`](../stackit-landingzone), shaped like its existing `variable "network"`, which -reuses this architecture as a component β€” see -[Composition](#composition-architectures-reuse-architectures). The SKE Starterkit is then a further -option that requires the AI option to be enabled. - -There is no layering conflict here, because the **SKE cluster is itself an offering inside a STACKIT -project**: the STACKIT LZ provisions the project, the SKE cluster building block turns it into a -Kubernetes platform, and that platform's landing zone provides the namespaces the AI components and -the starterkit need. Note the cluster building block does not exist in the hub yet β€” SKE clusters are -provisioned by foundation Terraform today (`platforms/ske/kubernetes/cluster.tf` in the -cloudfoundation repos), so moving it into the hub as `modules/stackit/ske` behind the `kubernetes` -option is a prerequisite β€” see [Option Layering](#option-layering-in-stackit-landingzone). - ## Getting Started ### Prerequisites -| Requirement | Description | -|----------------------|--------------------------------------------------------------------------------| -| Kubernetes landing zone | A meshStack landing zone providing namespaces β€” STACKIT SKE or Azure AKS. | -| Model backend | STACKIT AI Model Serving enabled, or an equivalent OpenAI-compatible endpoint. | -| meshStack instance | With Terraform/OpenTofu IaC runtime configured. | +| Requirement | Description | +|----------------------|--------------------------------------------------------------------| +| meshStack instance | With Terraform/OpenTofu IaC runtime configured. | +| `stackit-kubernetes` | Deployed, providing the SKE cluster, its HTTPS ingress with cert-manager, and a Kubernetes platform with a namespace landing zone. | +| Model backend | STACKIT AI Model Serving enabled, or an equivalent OpenAI-compatible endpoint such as Azure OpenAI. | +| DNS zone | A hostname for the LiteLLM gateway and the Langfuse UI, resolvable to the cluster's load balancer. | ### Deployment Order - +1. Deploy [`stackit-kubernetes`](../stackit-kubernetes). It provisions the SKE cluster with its + ingress and TLS, and registers a Kubernetes platform with a namespace landing zone. +2. Enable the model backend: STACKIT AI Model Serving, or an Azure OpenAI deployment. +3. Order the **AI Platform** building block once in the platform team's own workspace. It installs + LiteLLM and Langfuse into a namespace of that landing zone, registers LiteLLM as a meshStack + platform, and creates the AI landing zones and the tenant-facing `ai/litellm-team` definition. +4. Publish the AI landing zones, setting the budget and model allow-list each one grants. +5. Application teams order model access in their own projects. ## Shared Responsibilities @@ -548,17 +158,3 @@ option is a prerequisite β€” see [Option Layering](#option-layering-in-stackit-l | Stay within the granted budget and model allow-list | ❌ | βœ… | | Review own traces and evaluations in their Langfuse project | ❌ | βœ… | | Build and operate the AI application or assistant | ❌ | βœ… | - -## Scratch Notes - - - -- Demo components in scope: LiteLLM, Langfuse, STACKIT AI Model Serving. -- OpenWebUI was dropped from the core: it is an application, not shared platform infrastructure, and - its built-in user/group model permissions would be a second policy store competing with LiteLLM. Its - RAG half also duplicates the already-dropped RAGFlow. Kept as a candidate optional block (see open - question 5). -- Original demo ran on Scaleway; only the runtime and model-serving layers need swapping. -- Hub modules still missing for the platform components themselves β€” only `stackit/model-serving` is - scaffolded, as a minimal first version around `stackit_modelserving_token` (the only AI-specific - resource in the STACKIT provider, v0.88.0). diff --git a/reference-architectures/ai-platform/ai-platform-pluggable.dot b/reference-architectures/ai-platform/ai-platform-pluggable.dot deleted file mode 100644 index 53dab906..00000000 --- a/reference-architectures/ai-platform/ai-platform-pluggable.dot +++ /dev/null @@ -1,90 +0,0 @@ -/* - * AI Platform β€” the two replaceable layers. - * Conventions: .agents/references/diagrams.md β€” render with: task diagrams - */ -digraph ai_platform_pluggable { - rankdir=TB - compound=true - splines=ortho - forcelabels=true - bgcolor="white" - nodesep=0.55 - ranksep=0.85 - - node [shape=box style="rounded,filled" fontname="Helvetica" fontsize=11 - fillcolor="#ffffff" color="#a2abb8" penwidth=1.1 margin="0.20,0.11"] - edge [fontname="Helvetica" fontsize=9 fontcolor="#697180" color="#8b95a3" arrowsize=0.7] - - USER [label="πŸ‘€ Application team" fillcolor="#ffffff" color="#a2abb8"] - APPS [label="πŸ’¬ Tenant AI applications Γ—N\nbuilt by teams on the gateway API" - fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] - - subgraph cluster_mesh { - label=" meshStack" - labeljust=l - fontname="Helvetica" fontsize=12 fontcolor="#697180" - style="rounded" color="#d5dae0" - margin=18 - - BBD_COMP [label="πŸ“¦ litellm Β· langfuse BBDs\nsupportedPlatforms: kubernetes" fillcolor="#ecedfb" color="#9aa2e6"] - LZ [label="πŸ›¬ Landing Zone\nprovides the namespace" fillcolor="#ecedfb" color="#9aa2e6"] - BBD_MODEL [label="πŸ“¦ model-access BBD\none per provider, same output shape" fillcolor="#ecedfb" color="#9aa2e6"] - - { rank=same; BBD_COMP -> LZ -> BBD_MODEL [style=invis] } - } - - subgraph cluster_platform { - label=" AI Platform Β· Helm into a tenant namespace" - labeljust=l - fontname="Helvetica" fontsize=12 fontcolor="#697180" - style="rounded" color="#d5dae0" - margin=18 - - LITELLM [label="πŸšͺ LiteLLM\nvirtual keys Β· budgets Β· allow-list" fillcolor="#e5f2ea" color="#85bfa0"] - LANGFUSE [label="πŸ“ˆ Langfuse\ntracing Β· evaluation" fillcolor="#e5f2ea" color="#85bfa0"] - - { rank=same; LITELLM -> LANGFUSE [style=invis] } - - LITELLM -> LANGFUSE [xlabel="traces" style=dashed constraint=false] - } - - subgraph cluster_runtime { - label=" Runtime layer Β· any meshStack Kubernetes platform" - labeljust=l - fontname="Helvetica" fontsize=12 fontcolor="#697180" - style="rounded" color="#d5dae0" - margin=18 - - SKE [label="☸️ STACKIT SKE\nsovereign" fillcolor="#eef2f6" color="#93a7bb"] - AKS [label="☸️ Azure AKS" fillcolor="#eef2f6" color="#93a7bb"] - ANYK8S [label="☸️ Any conformant cluster" - fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] - - { rank=same; SKE -> AKS -> ANYK8S [style=invis] } - } - - subgraph cluster_models { - label=" Model layer Β· OpenAI-compatible backends" - labeljust=l - fontname="Helvetica" fontsize=12 fontcolor="#697180" - style="rounded" color="#d5dae0" - margin=18 - - STACKITM [label="🧠 STACKIT AI Model Serving\nsovereign" fillcolor="#eef2f6" color="#93a7bb"] - AZUREM [label="🧠 Azure OpenAI" fillcolor="#eef2f6" color="#93a7bb"] - VLLM [label="🧠 Self-hosted vLLM" - fillcolor="#f0f1f3" color="#cdd2d8" fontcolor="#99a1ac" style="rounded,filled,dashed"] - - { rank=same; STACKITM -> AZUREM -> VLLM [style=invis] } - } - - USER -> APPS [xlabel="builds and uses"] - APPS -> LITELLM [xlabel="calls the gateway"] - - BBD_COMP -> LITELLM [xlabel="deploys via Helm" style=dotted lhead=cluster_platform] - LZ -> SKE [xlabel="backed by" style=dotted lhead=cluster_runtime constraint=false] - BBD_MODEL -> LITELLM [xlabel="virtual key and model list" style=dotted constraint=false] - - LANGFUSE -> SKE [xlabel="namespace runs on" ltail=cluster_platform lhead=cluster_runtime] - LITELLM -> STACKITM [xlabel="routes by policy" lhead=cluster_models] -} diff --git a/reference-architectures/ai-platform/ai-platform-pluggable.svg b/reference-architectures/ai-platform/ai-platform-pluggable.svg deleted file mode 100644 index 6d1d60c3..00000000 --- a/reference-architectures/ai-platform/ai-platform-pluggable.svg +++ /dev/null @@ -1,179 +0,0 @@ - - - - -ai_platform_pluggable - - -cluster_mesh - -  meshStack - - -cluster_platform - -  AI Platform Β· Helm into a tenant namespace - - -cluster_runtime - -  Runtime layer Β· any meshStack Kubernetes platform - - -cluster_models - -  Model layer Β· OpenAI-compatible backends - - - -USER - -πŸ‘€ Application team - - - -APPS - -πŸ’¬ Tenant AI applications Γ—N -built by teams on the gateway API - - - -USER->APPS - - -builds and uses - - - -LITELLM - -πŸšͺ LiteLLM -virtual keys Β· budgets Β· allow-list - - - -APPS->LITELLM - - -calls the gateway - - - -BBD_COMP - -πŸ“¦ litellm Β· langfuse BBDs -supportedPlatforms: kubernetes - - - -LZ - -πŸ›¬ Landing Zone -provides the namespace - - - - -BBD_COMP->LITELLM - - -deploys via Helm - - - -BBD_MODEL - -πŸ“¦ model-access BBD -one per provider, same output shape - - - - -SKE - -☸️ STACKIT SKE -sovereign - - - -LZ->SKE - - -backed by - - - -BBD_MODEL->LITELLM - - -virtual key and model list - - - -LANGFUSE - -πŸ“ˆ Langfuse -tracing Β· evaluation - - - - -LITELLM->LANGFUSE - - -traces - - - -STACKITM - -🧠 STACKIT AI Model Serving -sovereign - - - -LITELLM->STACKITM - - -routes by policy - - - -LANGFUSE->SKE - - -namespace runs on - - - -AKS - -☸️ Azure AKS - - - - -ANYK8S - -☸️ Any conformant cluster - - - - -AZUREM - -🧠 Azure OpenAI - - - - -VLLM - -🧠 Self-hosted vLLM - - - - From 4a1cfd7f28ed5e07f60329c8a07ffb5518d82887 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 22:25:41 +0200 Subject: [PATCH 15/60] feat(kubernetes/ingress): consolidate cert-manager, HAProxy and the ClusterIssuer Five foundation units carry copies of certmanager.tf, haproxy.tf and cluster_issuer.tf, and the copies have drifted apart. This building block delivers the capability they share: a service in the cluster gets a public HTTPS URL with a valid certificate. The ClusterIssuer and the wildcard Certificate render through a Helm chart that lives in the module directory (chart = path.module). kubernetes_manifest would need a plan-time CRD schema lookup, which is the only reason foundations run addons/certmanager as a separate terragrunt unit today. Removing that lookup lets a foundation merge the two units into one. DNS-01 is optional and takes exactly one provider. The stackit branch installs the stackit-cert-manager-webhook chart and issues one wildcard certificate for the zone, which HAProxy then serves as its default TLS certificate. The route53 branch carries the native cert-manager solver shape but is not exercised, because AKS keeps per-hostname HTTP-01 for now. The module is sourced, never ordered, so it ships no meshstack_integration.tf and no backplane, and puts its user-facing readme in APP_TEAM_README.md. Co-Authored-By: Claude Opus 5 (1M context) --- .../ingress/buildingblock/.helmignore | 25 +++ .../ingress/buildingblock/APP_TEAM_README.md | 70 ++++++ .../ingress/buildingblock/Chart.yaml | 5 + .../ingress/buildingblock/README.md | 150 +++++++++++++ .../kubernetes/ingress/buildingblock/logo.png | Bin 0 -> 7309 bytes .../kubernetes/ingress/buildingblock/main.tf | 205 ++++++++++++++++++ .../ingress/buildingblock/outputs.tf | 30 +++ .../ingress/buildingblock/provider.tf | 15 ++ .../templates/cluster-issuer.yaml | 43 ++++ .../templates/wildcard-certificate.yaml | 18 ++ .../ingress/buildingblock/variables.tf | 151 +++++++++++++ .../ingress/buildingblock/versions.tf | 16 ++ 12 files changed, 728 insertions(+) create mode 100644 modules/kubernetes/ingress/buildingblock/.helmignore create mode 100644 modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md create mode 100644 modules/kubernetes/ingress/buildingblock/Chart.yaml create mode 100644 modules/kubernetes/ingress/buildingblock/README.md create mode 100644 modules/kubernetes/ingress/buildingblock/logo.png create mode 100644 modules/kubernetes/ingress/buildingblock/main.tf create mode 100644 modules/kubernetes/ingress/buildingblock/outputs.tf create mode 100644 modules/kubernetes/ingress/buildingblock/provider.tf create mode 100644 modules/kubernetes/ingress/buildingblock/templates/cluster-issuer.yaml create mode 100644 modules/kubernetes/ingress/buildingblock/templates/wildcard-certificate.yaml create mode 100644 modules/kubernetes/ingress/buildingblock/variables.tf create mode 100644 modules/kubernetes/ingress/buildingblock/versions.tf diff --git a/modules/kubernetes/ingress/buildingblock/.helmignore b/modules/kubernetes/ingress/buildingblock/.helmignore new file mode 100644 index 00000000..4758f993 --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/.helmignore @@ -0,0 +1,25 @@ +# The module directory is the chart directory, so everything that is not part of the chart has +# to be excluded here. Helm stores the packaged chart in a Kubernetes Secret β€” keeping it small +# is critical. + +# Terraform files and state +*.tf +*.tfstate +*.tfstate.backup +*.tfplan +.terraform/ +.terraform.lock.hcl + +# Terraform test files +*.tftest.hcl + +# Kubernetes credentials β€” never bundle into the chart +kubeconfig.yaml +kubeconfig-mock.yaml + +# Misc +.DS_Store +*.png +*.svg +README.md +APP_TEAM_README.md diff --git a/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md b/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md new file mode 100644 index 00000000..52e148ad --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md @@ -0,0 +1,70 @@ +Your services in this cluster get a public HTTPS URL with a certificate that browsers trust. The platform team runs an ingress controller and cert-manager for you, so all you add to your service is an Ingress object with a hostname. + +## 🎯 When to use it + +Use this building block when you: +- want to reach a service in the cluster from outside, over a real domain name instead of a port forward +- need TLS that browsers, mobile apps and API clients accept without a warning +- do not want to buy, renew or store certificates yourself + +## πŸ’‘ Usage examples + +**Example 1: Publish a web frontend** +You deploy a frontend Service in your namespace and add an Ingress for `shop.example.com` with the platform's ingress class. The controller starts routing traffic to your Service and the hostname answers over HTTPS right away. + +**Example 2: Expose an API for a partner** +Your team needs a stable HTTPS endpoint for a partner integration. You create an Ingress for `api.example.com`, hand the URL to the partner, and the certificate keeps renewing itself as long as the Ingress exists. + +## πŸ”§ How to use it + +Add an Ingress to your namespace and set the ingress class the platform team gave you: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: shop +spec: + ingressClassName: haproxy + rules: + - host: shop.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: shop + port: + number: 8080 +``` + +When the platform team runs a wildcard certificate for the cluster domain, that is all you need β€” the controller already serves a valid certificate for every hostname in the zone. + +For a hostname outside that zone, ask cert-manager for its own certificate. Add the ClusterIssuer annotation and a `tls` section: + +```yaml +metadata: + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + tls: + - hosts: + - shop.example.com + secretName: shop-tls +``` + +The hostname has to resolve to the ingress load balancer before Let's Encrypt can validate it, so create the DNS record first. + +## πŸ“Š Shared Responsibility + +| Responsibility | Platform Team | Application Team | +|---|:---:|:---:| +| Run the ingress controller and its load balancer | βœ… | ❌ | +| Run cert-manager and the Let's Encrypt ClusterIssuer | βœ… | ❌ | +| Renew the wildcard certificate for the cluster domain | βœ… | ❌ | +| Create DNS records for the cluster domain | βœ… | ❌ | +| Create the Ingress object and pick the hostname | ❌ | βœ… | +| Keep the backend Service healthy and reachable | ❌ | βœ… | +| Create DNS records for hostnames outside the cluster domain | ❌ | βœ… | +| Authentication and authorization inside the application | ❌ | βœ… | diff --git a/modules/kubernetes/ingress/buildingblock/Chart.yaml b/modules/kubernetes/ingress/buildingblock/Chart.yaml new file mode 100644 index 00000000..03f59feb --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: meshstack-ingress-issuer +description: ClusterIssuer and wildcard Certificate for the meshStack Kubernetes ingress building block. +type: application +version: 1.0.0 diff --git a/modules/kubernetes/ingress/buildingblock/README.md b/modules/kubernetes/ingress/buildingblock/README.md new file mode 100644 index 00000000..28ef1482 --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/README.md @@ -0,0 +1,150 @@ +--- +name: Kubernetes Ingress with TLS +supportedPlatforms: + - kubernetes +description: Installs cert-manager, the HAProxy ingress controller and a Let's Encrypt ClusterIssuer so every service in the cluster can get a public HTTPS URL with a valid certificate. +# All credentials β€” cluster and DNS provider β€” arrive as inputs, so there is nothing to set up +# on the cloud side. +requiresBackplane: false +--- + +# Kubernetes Ingress with TLS + +This module delivers one capability: **services in the cluster get a public HTTPS URL with a valid certificate.** It installs the three pieces that capability needs, in one Terraform run: + +1. **cert-manager** β€” requests and renews certificates from Let's Encrypt. +2. **HAProxy ingress controller** β€” terminates TLS and routes traffic to Services, behind a cloud load balancer. +3. **A Let's Encrypt ClusterIssuer** β€” the issuer application teams reference from their Ingress objects. + +This documentation is intended as a reference for cloud foundation or platform engineers using this module. + +## Why the ClusterIssuer runs through a local Helm chart + +A `kubernetes_manifest` resource looks the CRD schema up at plan time. cert-manager installs the `ClusterIssuer` CRD, so the schema does not exist yet during the first plan. Foundations worked around this by putting the ClusterIssuer in a second Terragrunt unit that runs after cert-manager. + +This module renders the ClusterIssuer through a Helm chart that lives in the module directory itself (`chart = path.module`). Helm applies the manifests at apply time and never asks Terraform for a schema, so cert-manager and the ClusterIssuer fit into a single unit. That is why the module directory carries a `Chart.yaml`, a `templates/` directory and a `.helmignore` that keeps every Terraform artifact out of the packaged chart. + +## Two ways to get certificates + +**HTTP-01, per hostname (default).** Leave `dns01` unset. cert-manager solves the ACME challenge over the ingress itself, so every hostname an application team asks for gets its own certificate. The hostname has to resolve to the load balancer before issuance can finish. + +**DNS-01, one wildcard for the whole zone.** Set `dns01` with a `zone_name` and exactly one provider. The module then creates a single `Certificate` for `*.` in `haproxy_namespace` and points HAProxy's `controller.defaultTLSSecret.secret` at the resulting secret. HAProxy serves that certificate for every host that brings none of its own, so a new application needs no certificate request at all. The certificate lives in the long-lived HAProxy namespace, so tearing an application namespace down never takes it with it. The HTTP-01 solver stays in place next to the DNS-01 one for hostnames outside the zone. + +### DNS-01 providers + +| Provider | Extra chart | Status | +|---|---|---| +| `stackit` | `stackit-cert-manager-webhook` | Implemented and exercised. | +| `route53` | none β€” cert-manager solves Route53 natively | Implemented, **not exercised**. | + +STACKIT DNS has no built-in cert-manager solver, so the module installs the [stackit-cert-manager-webhook](https://github.com/stackitcloud/stackit-cert-manager-webhook) chart into `cert_manager_namespace` and stores the service account key in the secret the webhook mounts. The ClusterIssuer then calls it with `solverName: stackit` and `groupName: acme.stackit.de`. + +The `route53` branch is a documented shape rather than a tested path. AKS foundations still issue per-hostname HTTP-01 certificates today, so nobody has run this branch against a live hosted zone. Treat it as a starting point and verify it before you rely on it. + +## What this module replaces + +Five foundation units carried copies of the same `certmanager.tf`, `haproxy.tf` and `cluster_issuer.tf`. The copies had drifted: + +| | SKE | AKS | +|---|---|---| +| cert-manager version | `v1.20.0` | `v1.19.4` | +| `--certificate-request-minimum-backoff-duration=1m` | present | missing | +| Service annotations | none | Azure health probe path | +| ACME contact | `ske@meshcloud.io` | `platform@likvid-bank.com`, `devops-platform@meshcloud.io` | + +The module defaults match the SKE copy, which is the most current one. AKS callers set `haproxy_service_annotations` and get the newer cert-manager and the retry-backoff tuning along the way. + +## Notes for platform engineers + +- **Providers.** Only `kubernetes` and `helm`. No cloud provider ever enters this module, so it works on SKE, AKS and anything else that speaks the Kubernetes API. Cloud-specific behaviour arrives as strings, mainly through `haproxy_service_annotations`. +- **Sourced, not ordered.** There is no `meshstack_integration.tf` and no `backplane/`. Foundations source `buildingblock/` from a Terragrunt unit, and reference architectures source it from their own building block. +- **Permissions.** The token in `var.token` needs cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC. +- **DNS records.** Point your DNS A record at the `haproxy_lb_ip` output. Nothing can be issued or served before that record resolves. +- **First apply.** HAProxy comes up before the wildcard certificate is issued. Until the secret exists, HAProxy serves its own self-signed certificate for unmatched hosts and picks the real one up as soon as cert-manager writes it. + +## Usage + +```hcl +module "ingress" { + source = "github.com/meshcloud/meshstack-hub//modules/kubernetes/ingress/buildingblock?ref=main" + + cluster_endpoint = var.cluster_endpoint + cluster_ca_certificate = var.cluster_ca_certificate + token = var.token + + acme_email = "ske@meshcloud.io" + + dns01 = { + zone_name = "likvid.stackit.run" + stackit = { + project_id = var.stackit_project_id + service_account_key = var.stackit_service_account_key + } + } +} +``` + + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [helm](#requirement\_helm) | >= 3.0.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.38 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.haproxy](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.issuer](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.stackit_cert_manager_webhook](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace_v1.cert_manager](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | +| [kubernetes_namespace_v1.haproxy_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | +| [kubernetes_secret_v1.route53_dns01](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [kubernetes_secret_v1.stackit_dns01](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [kubernetes_service_v1.haproxy_controller](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/service_v1) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [acme\_email](#input\_acme\_email) | Contact address Let's Encrypt uses for expiry warnings and account recovery. | `string` | n/a | yes | +| [acme\_private\_key\_secret\_name](#input\_acme\_private\_key\_secret\_name) | Name of the secret in which cert-manager stores the ACME account private key. | `string` | `"letsencrypt-prod-account-key"` | no | +| [acme\_server](#input\_acme\_server) | ACME directory URL. Point this at https://acme-staging-v02.api.letsencrypt.org/directory while you test, because the production endpoint has strict rate limits. | `string` | `"https://acme-v02.api.letsencrypt.org/directory"` | no | +| [cert\_manager\_crds\_keep](#input\_cert\_manager\_crds\_keep) | Keep the cert-manager CRDs when the Helm release is destroyed. Keeping them preserves existing Certificate and ClusterIssuer objects across a reinstall. | `bool` | `false` | no | +| [cert\_manager\_extra\_args](#input\_cert\_manager\_extra\_args) | Extra command line arguments for the cert-manager controller. | `list(string)` | 
[
  "--certificate-request-minimum-backoff-duration=1m"
]
| no | +| [cert\_manager\_namespace](#input\_cert\_manager\_namespace) | Namespace for cert-manager and, when DNS-01 runs through STACKIT, for the STACKIT cert-manager webhook. The webhook chart expects both in the same namespace. | `string` | `"cert-manager"` | no | +| [cert\_manager\_version](#input\_cert\_manager\_version) | Version of the cert-manager Helm chart. See https://github.com/cert-manager/cert-manager/releases. | `string` | `"v1.20.0"` | no | +| [cluster\_ca\_certificate](#input\_cluster\_ca\_certificate) | Cluster CA certificate, base64 encoded. | `string` | n/a | yes | +| [cluster\_endpoint](#input\_cluster\_endpoint) | IP address or hostname of the cluster control plane, without the https:// scheme. | `string` | n/a | yes | +| [cluster\_issuer\_name](#input\_cluster\_issuer\_name) | Name of the ClusterIssuer. Application teams reference it from the cert-manager.io/cluster-issuer annotation on their Ingress. | `string` | `"letsencrypt-prod"` | no | +| [dns01](#input\_dns01) | Enables a wildcard certificate for zone\_name via DNS-01. Set exactly one provider. Null keeps HTTP-01 per-hostname issuance. | 
object({
    zone_name = string
    stackit   = optional(object({ project_id = string, service_account_key = string }))
    route53   = optional(object({ hosted_zone_id = string, access_key_id = string, secret_access_key = string, region = optional(string, "eu-central-1") }))
  })
| `null` | no | +| [haproxy\_namespace](#input\_haproxy\_namespace) | Namespace for the HAProxy ingress controller. The wildcard certificate is created here as well, so its secret survives the teardown of any application namespace. | `string` | `"haproxy-ingress"` | no | +| [haproxy\_release\_name](#input\_haproxy\_release\_name) | Helm release name of the HAProxy ingress controller. The chart names the controller Service '-kubernetes-ingress'. | `string` | `"haproxy"` | no | +| [haproxy\_replica\_count](#input\_haproxy\_replica\_count) | Number of HAProxy ingress controller replicas. | `number` | `2` | no | +| [haproxy\_service\_annotations](#input\_haproxy\_service\_annotations) | Annotations on the HAProxy controller Service. The cloud provider reads them to configure the
load balancer. Two values matter in practice:
- AKS needs `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path = "/healthz"`.
- STACKIT uses `lb.stackit.cloud/internal-lb` to keep the load balancer off the public internet. | `map(string)` | `{}` | no | +| [haproxy\_service\_type](#input\_haproxy\_service\_type) | Service type of the HAProxy ingress controller. | `string` | `"LoadBalancer"` | no | +| [haproxy\_timeout](#input\_haproxy\_timeout) | Seconds to wait for the HAProxy Helm release to become ready. The default of 20 minutes covers the time a cloud provider takes to provision the load balancer. | `number` | `1200` | no | +| [haproxy\_version](#input\_haproxy\_version) | Version of the haproxytech/kubernetes-ingress Helm chart. See https://github.com/haproxytech/helm-charts/blob/main/kubernetes-ingress/Chart.yaml. | `string` | `"1.49.0"` | no | +| [ingress\_class\_name](#input\_ingress\_class\_name) | Name of the IngressClass the controller serves. The HTTP-01 solver of the ClusterIssuer uses the same name. | `string` | `"haproxy"` | no | +| [stackit\_webhook\_version](#input\_stackit\_webhook\_version) | Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook. | `string` | `"0.4.10"` | no | +| [token](#input\_token) | Token of the service account this module runs as. It needs cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC. | `string` | n/a | yes | +| [wildcard\_certificate\_name](#input\_wildcard\_certificate\_name) | Name of the wildcard Certificate and of the secret it writes, both in haproxy\_namespace. Only used when dns01 is set. | `string` | `"wildcard-tls"` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [cluster\_issuer\_name](#output\_cluster\_issuer\_name) | Name of the ClusterIssuer an application references from the cert-manager.io/cluster-issuer annotation on its Ingress. | +| [haproxy\_lb\_ip](#output\_haproxy\_lb\_ip) | External IP of the HAProxy LoadBalancer service. Point your DNS A record here before TLS provisioning can complete. | +| [haproxy\_namespace](#output\_haproxy\_namespace) | Namespace of the HAProxy ingress controller and of the wildcard certificate secret. | +| [ingress\_class\_name](#output\_ingress\_class\_name) | Name of the IngressClass an application puts on its Ingress to be served by this controller. | +| [wildcard\_certificate\_secret\_name](#output\_wildcard\_certificate\_secret\_name) | Name of the secret in haproxy\_namespace holding the wildcard certificate. Null when dns01 is not set. | + diff --git a/modules/kubernetes/ingress/buildingblock/logo.png b/modules/kubernetes/ingress/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ff97e71f2dc44787031230d84a3bb702e584909c GIT binary patch literal 7309 zcmXw8c{J4D`+v`v$yTvIU2rhSb-n=+D+B#cmJGCiTHmIKH+NJYEa?e8zuK_jZ9_jrr)SbI`yZ39l zb*npmIOy3XbD%=g=c|@=-EQl839ScGdS&c&t{kCG{_tB_qpD4;_-^v%eo!_D{(pI1 z=-mtztto0wA2+eP-D&DBJ*vk$RUJDtygn=2wkjT_C|EQEUint=VzTo6@2mI67nj#J zA^IL2?fs1Iugubl@#3nPW8niw13Kl*>TWz3pPgr|tZocX{H<^Mec{>{_t-C~d4rd4 z|9soQ7#d-YPp!7lmgESvp3&W3ellf^tIO)9TtYq=`S;5lc$NS7=jb%+M(JqJz+zkX z+ySS?n#Ny&aqSi-o5PcPr1hU)FaCMNvpy)XN8k3ft>632qAsTPq~-KJEdQx=@R_96 zgPZrheEYHZqGrVHY-96>UlJPRqi!`jR0_q_i~S?sKX^8{m+)k}Las+{+0D%^E1o}L z?*u8ip1*eQ)u;B)>l<4g=l>xOr@8ZgSN{F|ke2h=PW-oinD=!xzF7#`)oyKJ<{bOu z&;5C#r}gKK1LP)npTeIy5eoHx?&+ahf2OrReG((tGm*uWiYBsB`v>9XHrH!Z0*@ z%YrXbCE(i83YdB?X8WlW<~u?_NFOif_fy!zN537c_ZAtf30S5G2CR3*YnZ4Y>(0&;1`6=7dH+Ij%=l)h0HhwoVK-JN2IB1RhJU8~XBN{Bv zJz_JKk2yYZb7Le^42%Uowze1)P_&!>a&J-P+YHDQgT}SaoraiT_ylLo8a<=^f{;vk zBNV`BeNxs_j;JA{l^TTje-+44WYnEiI!`?qoDLp`hbAQSu6=|k9eVSP6?MdRF!z!W zVZ;e1{+?lEo$KK4OB6%Kj`$kyPu{{hR2(AuQ{M+UZgk&kD*95bHqrEp*7P zskKXtj~Y=X`}-(j^4N5VX@*vcQDj!=3zek`3v1MacJ}4g{c*@SCze}`H7cfeZ0(Y` zN+=T0GfvF7iEgtUXGo$ugbm=56^DDISD-CV`q zLi_`))^VUFOoGoSXE2}I{t`}uFCdkFh&G(o8QG0y6DPLnLsInD)5zVyU`5n7+iCI@ z^hf^zHZhyV!|0jbOphL&U!_!wZB>rH?2o7B@nP+z80sa%PKd8UZByP8vU0BiZxB)IfSBGdiDyvFGi^(`1H5KsjM0gr@qwoKA=4*h2oi%(?3fh6xEE}X!0Ltc&h?Lozm%ivWv`H4gtb=y5h} zBqs*(ws(u5ma2krnaMn4B{b(gXcqUa{EfX0>$cej&Cm`-=GQ~t~3>0z^u~rI3=t=xwxpDI36?{~;JBXsRE{xX6cKn!*gkk~ylyP(Fj= zKvve3PbRoa7Iit;Cdy=z-te$;4lFwOa@2#?!&R=ZiPEUfNxI2}yNfscIr66mA{aH; z{zg7SLAfNeb%Gq{&T94}B{m#_nq8^ru(Ep@dWCW=wD6Nd2sUK;t!dj%iorDTu{XUo zubN}^1nOC#ys+qdRT!)ZMbbQl^sxQ2LXpX)Cbx8Tbm3q7Muu!!jX^+D(d6Ru@t8pj`y2toH4J zG1ZDo7fsoe*H8<~E!`b%;K7zen)@RqYTjgx<=^bdX6EHoFr%XP+}q zsNv~@RWYPC*xYb2NkjU9$+5y#;bL^Mn(6hJUZ=Nlqa26^I1VAHoVX@;foZU-+ABG3+p>cJjR*4$L|~vLTm!`j zGaYXHDNej69cD21n+0-lUJyh$gZ}KN20tACa(8RXTQC&h;~n@M_zTF*ROq4~*-FQRsS)E?#^ z=Z23)U7Y9`TQ~58)CqLq8|G}wfPMiAIHLF|u0_Ojo}z|&#A##F6l>&vV3?UbWBUvN z>P%w8MCL>)~PloeKIPrS3&>H2i)>r^RP>e1{Phweb!dXoEpE$0^ z`6JRiZ1!jg?jA@f<_%LcZ9cTAz@$W?PILCeKmOpIFys)ndza?aOhwW;H8?pE6K^Gf zHbsn7NY}~k$m7!a6;bwCauTOee{565d{2^Mb0jK#r}>yXHfI^vO)>Fg?D<9RAaSys zcq6b&Owd1KwF)xca*fU*z@4P1PB?ic?}$)STD}2hV&-Mixjxw>4FAdf8s z*W+(&V{$UBdWKS|JUCMyghjU=NpKYz;F~;4<_$4LeD4%O=>VXBC*E>Z`!LF|8`d#H`PEJ_OeLi!;v+EX=j%n~owmt56Z zBz#8)dLpeWL==%H3K?*^dQo2C8SISPu5gQWRs zoC@(E4(D#J(5}IX!6n!ULaknC$!o+T3&V1QmKg~xB4f+2CkX}1WNR*Tvok%uo@z7Y zJf*qscRUGRB0)Ni976h8k(YkU;m1g3zlyvO2vMjOw#XDo{4C(UDKj=D4Sg<5U z_f0oZ4>lj!!ugLDSKYi0OD@6^nAdl{m(5LtZ_nS5BLnHxD^JY0^c-|;&Yr}I!IV7i zo$QHaMb2o0%(o7D*>l(m*W$nj{f5)tqRd9P3=wjf^ez@f-Xi>19eYJkFQSq8)cz z5-kJx6WR|bfc)-ud4t3O@%=We>A zR)|asj7V=wPZ}jyn{nmLxG9$kWDc*8omZ+W@+iy7*YN4eoFPBoEq{^BORhHai|_%& z=wmDrEx8?t&k)YP&)ws)`IH?-4Ke7_&v;!%xY`|~e?><*6Y2JA10M-Pg13GrK_?Pz zNgn+hgdl~XCnku)GAFjQJrdPENV@s5)t}4ZMtTRCv9Rc+Q39HuI=;4`>ObTo>_9T> zC_OD-o=2hVl;d!esR@9;8k3LEyMCx#*gB?_v>fBO=WEeA%ZyIeh5%QCB>ISD{%lu6 zQGg@^b_%#_nyS)8!Yc z2%Xjut`$7`&4%0k$vLKllMiK^GALc%Jm>+HveAiWpHzcnBclZk@1V9!20tinpOc;MuZsxDjaz# zUL1YK|BG_i{FLj^M&3CEaoL0Oiic=I^i0EuL>h*3X!(l*#b0bWq#(fO(5Vy=W1ekwR|h6Xi+T`p1q0aSg!6 zUB(Wil}tn)C<%@)$YT;q#T5mz2|9(Vh|-0x+R_E9r_;T_&C@_~_jZx@5xnPb1lqL2 zS04*f0#+WjRr#W0-$YkV2@1?pID2uiV*C;kQ^7j0I7Rj>b7LR)(mDF+y_GBID1dfw zFw%Rl*SB4?X@jp4vk9k`d24G^3A3Thl6CrWT9%3pJ6>J#%rYL2*_!Q{MuSsSj}E#2 zgSWk(xHKImv+P-flG{r?~lMpR`}As16Rj^l2YQ@ zC*)r78o~Ku6z9{6u<~_+mtel(tq?>?!6*L4KhTC|U3d>iwftD!n>cuc`sMg3Chl4F zPNP5IST6Pb`!%9aS=1V0U`2tIeVK24m*~c-HmQM|mc>*>h>|GH*G3$^Umo*xcL}sY z_p$z#U9_^&J9+zrWEig_BBkq-cr6l7MxABOSFiLTVGkio`qYLu6Y6jm@m?|D#9-$!88U>PceE|q)~Vy{Q}=;p^X zP=4QmtDI0xwa7Dur*U?esfm%2>v zzk1s~2D!YqUUb{rfnfO~fh>oYoFKVJJ@+7SG5%&%?}gnS79yOADTK!+jTWkK1dxoGW{K6u!_V`a{ry zG*iAsfXbPvS9_0Zp~EM*0DtdC)%*T!f(K}d>Y`{@6>nD#0Fivg)$vQ1=k1c!*&&dj3WZa zsB3f+CWCP8GrvgwQ&!Q-6VFCV7vcBE8cR*!yg|o)fj(g2b3-JZu3ro#il@+bU|*4L zPX3%82IrNmlK;!GcX+#HX}AKS2{A_rHc_gO$EvC^<~pnUdhDR@QM6L;9vI6v2#Ipd zTI;MK%DXNrpK?xzz{2*ug+N*18)`o~siqj>)K$AiApRnu&v)ITz)$T-iaTK!u`R4? zE3}6(St4V$QC`~aC^Rws2K=P5af>AE-n1>anhBqdEb$*k2CJ;I1m!$zljdK|cBxAl zg0In}yKL)G!lSU(bdI)1u{Dkk_BOqrKz})7#(Y&)RgI>>?a9-TI)ntN)Rk^m>Px7z z9riON*}AbhugDaGti|C3J-JwLVpB$M1$aUzdpj{Xxy19m0yJ>3v`JRi1qJ6uN~KQ0 z(^Kesg(_5#ftSd?I!N$6zQQ98xK7id5`|!QQ);^t{W^c8THGmf2QKYT+}G{|a5qk> zeGiwygUVIPF7Iibha2-ZZmmVJ);!+$1+K4*-xmnAkD2w}gF#B)u_BiAxnPzkmeTP%WyMX=*<}I4f_im$J4K%ZMX}LIC`P&BV`p>-& z*5zLwCq=)4+(b}2&xD7@jdM;;j8qu+;va;!b~1(E>__$0qrD3hsGS1P3Vw+#6@8Nd z54`bHg7r$Q{|DTQ%6(K38mGUcc!8GEZ@FXUX?W|k*=cLMPw&RU-zc zr91gRuU$6@+S$;~!HY{?*MfJcpoHx_a*3d8!}4c4tHCXA3&yc=*9ts4LCfBwH1$%s z41U?FLUKjI-aMvnDTxt@1vcVTt6OuS?yioY`#VOsI{&%wLDl4zoE+duoupg2}wl7CnuO`!21V(JyKV` zy2`Mt-}a@0ZZ~fvc-`OJlO0_ja;tXhuuCKtc2*(TC*FWEO;MLJO*LTfw|l*G(NkVy z7LHICC0Uxl|FjZ3jDDxHvIKmqHa(uH7X0^wc<&33X=V-9y*f}*@Ez;2`Ka4i z=0U8kH!A-LIU=&iII#}OEW}WIp?gHxd;#e)^y@fu<(f%LU|lRH1eL4L?|Xc@b|p>F z`l!{-xGK@W=rtgq%KuJKj9O|3Wjby430rL~3U>q%)i2WNJzISX!Ny}h97*DE8MgxtEXV)bQzdMDlxH?q1+_H!u%UJGtUyVi zIF)&{bzODg`_i8p+GC%S)4~_Gy=EhX{eH*16LTS(gRBiN)E$t2kS}|MZ z`SW?248OXm==QF4>MZ|Knk-85BW@P4I$)ZX4v5BKU;7IjkFOBseNkcAFtNk)Nl|=a zDioGxNk#+IBKXpNTa`w*KiPWX&*$t{Nr{WW7&Yn==V#mQQbW#G)K0B~X-uhfg3VQ) zkM|Jx{&eIxrVBPs;ooZlgkeY9HkycO)vW878wBsM+2B%sC_yd?F$tap>n>`LC}@(NN)%-AqfQ!Xw(qF4p%bXyr8qloxIY@z$y%dEM;Tah zu0WtBf3I}R?+xCw{zNKq@EZ`^JqJLoJT^23Y!tAip!ZV71w9+Q z4^vHbQaEm+xI?COTCeS9xoM?F+&p=Nky87s&QrxOOtl#^N|CS~ovjr`niH9)zUzJ( zuiT-^L^{SCf4`caW(d6pw@wxObzH!W5eUH8ojOH1c!>oLNWO(^K<#jI#E;$?M%;(i z)+!?>_LImvino{Gt6`s<2H2M~DUTa!IsIIpy5a8;Ly3C#!Q9;XXFl+tH7c*wFF3y! zz7XxQMz}&FRHxRZc5#}PY2rRC?L}Ewt{HcmeRU;8g=y_ubJSR86MindNuCGBJWd;m;gKe>qe&>!?4QG#g#|G_&Dd1}hKGOyryY_j4 zk-M?)Uft8_WrA14XkSPNbK8-0=8a9y@6cntJed84A{=YHPf77N0X&`(-S3RuGWUgX!00voG+FQIj H?3?_5mi62J literal 0 HcmV?d00001 diff --git a/modules/kubernetes/ingress/buildingblock/main.tf b/modules/kubernetes/ingress/buildingblock/main.tf new file mode 100644 index 00000000..468ac221 --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/main.tf @@ -0,0 +1,205 @@ +locals { + # var.dns01 is sensitive as a whole, so every expression derived from it carries the sensitivity + # mark. count arguments, resource names and outputs reject a marked value, so unmark the plain + # facts β€” is DNS-01 on, which provider, which zone β€” while the credentials keep their mark. + # nonsensitive() rejects an argument that carries no mark, so try() falls back to the value. + dns01_enabled = try(nonsensitive(var.dns01 != null), var.dns01 != null) + dns01_stackit_enabled = local.dns01_enabled ? try(nonsensitive(var.dns01.stackit != null), var.dns01.stackit != null) : false + dns01_route53_enabled = local.dns01_enabled ? try(nonsensitive(var.dns01.route53 != null), var.dns01.route53 != null) : false + dns01_zone_name = local.dns01_enabled ? try(nonsensitive(var.dns01.zone_name), var.dns01.zone_name) : null + + # The chart derives the controller Service name from the release name. + haproxy_service_name = "${var.haproxy_release_name}-kubernetes-ingress" +} + +resource "kubernetes_namespace_v1" "cert_manager" { + metadata { + name = var.cert_manager_namespace + } +} + +resource "helm_release" "cert_manager" { + name = "cert-manager" + namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name + repository = "https://charts.jetstack.io" + chart = "cert-manager" + version = var.cert_manager_version + + create_namespace = false + wait = true + timeout = 300 + + values = [ + yamlencode({ + crds = { + enabled = true + keep = var.cert_manager_crds_keep + } + extraArgs = var.cert_manager_extra_args + }) + ] +} + +# The webhook mounts the STACKIT service account key from a secret, so the secret has to sit in +# the same namespace as the webhook pod. +resource "kubernetes_secret_v1" "stackit_dns01" { + count = local.dns01_stackit_enabled ? 1 : 0 + + metadata { + name = "stackit-sa-authentication" + namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name + } + + data = { + "sa.json" = var.dns01.stackit.service_account_key + } +} + +# cert-manager has no built-in solver for STACKIT DNS. This chart registers one under the +# acme.stackit.de API group, which the ClusterIssuer then calls as solverName "stackit". +resource "helm_release" "stackit_cert_manager_webhook" { + count = local.dns01_stackit_enabled ? 1 : 0 + + name = "stackit-cert-manager-webhook" + namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name + repository = "https://stackitcloud.github.io/stackit-cert-manager-webhook" + chart = "stackit-cert-manager-webhook" + version = var.stackit_webhook_version + + create_namespace = false + wait = true + timeout = 300 + + values = [ + yamlencode({ + groupName = "acme.stackit.de" + certManager = { + namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name + serviceAccountName = "cert-manager" + } + stackitSaAuthentication = { + enabled = true + secretName = kubernetes_secret_v1.stackit_dns01[0].metadata[0].name + } + }) + ] + + depends_on = [helm_release.cert_manager] +} + +# The route53 solver is built into cert-manager, so it needs no extra chart. It only needs the +# secret access key handed to it through a secret reference. +resource "kubernetes_secret_v1" "route53_dns01" { + count = local.dns01_route53_enabled ? 1 : 0 + + metadata { + name = "route53-dns01-credentials" + namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name + } + + data = { + "secret-access-key" = var.dns01.route53.secret_access_key + } +} + +resource "kubernetes_namespace_v1" "haproxy_ingress" { + metadata { + name = var.haproxy_namespace + } +} + +# The ClusterIssuer and the wildcard Certificate are custom resources whose CRDs only exist once +# cert-manager is installed. Helm renders and applies them without a plan-time schema lookup, +# which is what kubernetes_manifest would need β€” that lookup is the reason foundations had to run +# the ClusterIssuer as a separate Terraform unit. +resource "helm_release" "issuer" { + name = "ingress-issuer" + namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name + chart = path.module + + atomic = true + wait = true + timeout = 300 + + values = [ + yamlencode({ + clusterIssuer = { + name = var.cluster_issuer_name + email = var.acme_email + server = var.acme_server + privateKeySecretName = var.acme_private_key_secret_name + } + ingressClassName = var.ingress_class_name + dns01 = { + zoneName = local.dns01_zone_name + stackit = local.dns01_stackit_enabled ? { + projectId = var.dns01.stackit.project_id + } : null + route53 = local.dns01_route53_enabled ? { + region = var.dns01.route53.region + hostedZoneID = var.dns01.route53.hosted_zone_id + accessKeyID = var.dns01.route53.access_key_id + secretAccessKeySecretName = kubernetes_secret_v1.route53_dns01[0].metadata[0].name + secretAccessKeySecretKey = "secret-access-key" + } : null + } + wildcardCertificate = { + enabled = local.dns01_enabled + name = var.wildcard_certificate_name + namespace = kubernetes_namespace_v1.haproxy_ingress.metadata[0].name + secretName = var.wildcard_certificate_name + } + }) + ] + + depends_on = [ + helm_release.cert_manager, + helm_release.stackit_cert_manager_webhook + ] +} + +resource "helm_release" "haproxy" { + name = var.haproxy_release_name + namespace = kubernetes_namespace_v1.haproxy_ingress.metadata[0].name + repository = "https://haproxytech.github.io/helm-charts" + chart = "kubernetes-ingress" + version = var.haproxy_version + + create_namespace = false + timeout = var.haproxy_timeout + + values = [ + yamlencode({ + controller = merge( + { + replicaCount = var.haproxy_replica_count + ingressClass = var.ingress_class_name + ingressClassResource = { name = var.ingress_class_name } + service = { + type = var.haproxy_service_type + annotations = var.haproxy_service_annotations + } + }, + # HAProxy serves this certificate for every host that brings no certificate of its own. + # controller.defaultTLSSecret.secretNamespace defaults to the release namespace, which is + # where the wildcard Certificate writes its secret, so only the name has to be set. + # Without DNS-01 the chart default stays in place and HAProxy keeps its self-signed + # certificate for unmatched hosts. + local.dns01_enabled ? { defaultTLSSecret = { secret = var.wildcard_certificate_name } } : {} + ) + }) + ] + + depends_on = [helm_release.issuer] +} + +# The cloud provider assigns the load balancer address after HAProxy is up. Foundations point +# their DNS A records at it. +data "kubernetes_service_v1" "haproxy_controller" { + metadata { + name = local.haproxy_service_name + namespace = kubernetes_namespace_v1.haproxy_ingress.metadata[0].name + } + + depends_on = [helm_release.haproxy] +} diff --git a/modules/kubernetes/ingress/buildingblock/outputs.tf b/modules/kubernetes/ingress/buildingblock/outputs.tf new file mode 100644 index 00000000..571c3d4e --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/outputs.tf @@ -0,0 +1,30 @@ +output "haproxy_lb_ip" { + description = "External IP of the HAProxy LoadBalancer service. Point your DNS A record here before TLS provisioning can complete." + value = data.kubernetes_service_v1.haproxy_controller.status[0].load_balancer[0].ingress[0].ip +} + +output "ingress_class_name" { + description = "Name of the IngressClass an application puts on its Ingress to be served by this controller." + value = var.ingress_class_name + + depends_on = [helm_release.haproxy] +} + +output "cluster_issuer_name" { + description = "Name of the ClusterIssuer an application references from the cert-manager.io/cluster-issuer annotation on its Ingress." + value = var.cluster_issuer_name + + depends_on = [helm_release.issuer] +} + +output "haproxy_namespace" { + description = "Namespace of the HAProxy ingress controller and of the wildcard certificate secret." + value = kubernetes_namespace_v1.haproxy_ingress.metadata[0].name +} + +output "wildcard_certificate_secret_name" { + description = "Name of the secret in haproxy_namespace holding the wildcard certificate. Null when dns01 is not set." + value = local.dns01_enabled ? var.wildcard_certificate_name : null + + depends_on = [helm_release.issuer] +} diff --git a/modules/kubernetes/ingress/buildingblock/provider.tf b/modules/kubernetes/ingress/buildingblock/provider.tf new file mode 100644 index 00000000..7002f69d --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/provider.tf @@ -0,0 +1,15 @@ +provider "kubernetes" { + host = "https://${var.cluster_endpoint}" + cluster_ca_certificate = base64decode(var.cluster_ca_certificate) + token = var.token +} + +# The helm provider talks to the same control plane with the same credentials, so namespaces, +# Helm releases and the resources the charts render all land in one cluster without extra wiring. +provider "helm" { + kubernetes = { + host = "https://${var.cluster_endpoint}" + cluster_ca_certificate = base64decode(var.cluster_ca_certificate) + token = var.token + } +} diff --git a/modules/kubernetes/ingress/buildingblock/templates/cluster-issuer.yaml b/modules/kubernetes/ingress/buildingblock/templates/cluster-issuer.yaml new file mode 100644 index 00000000..a92a958e --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/templates/cluster-issuer.yaml @@ -0,0 +1,43 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ .Values.clusterIssuer.name }} +spec: + acme: + email: {{ .Values.clusterIssuer.email | quote }} + server: {{ .Values.clusterIssuer.server | quote }} + privateKeySecretRef: + name: {{ .Values.clusterIssuer.privateKeySecretName }} + solvers: + {{- with .Values.dns01.stackit }} + # DNS-01 through the STACKIT webhook. It answers for the whole zone, which is the only way + # to get a wildcard certificate β€” HTTP-01 can never validate one. + - dns01: + webhook: + solverName: stackit + groupName: acme.stackit.de + config: + projectId: {{ .projectId | quote }} + selector: + dnsZones: + - {{ $.Values.dns01.zoneName | quote }} + {{- end }} + {{- with .Values.dns01.route53 }} + # cert-manager solves DNS-01 against Route53 itself, so this branch needs no extra chart. + - dns01: + route53: + region: {{ .region | quote }} + hostedZoneID: {{ .hostedZoneID | quote }} + accessKeyID: {{ .accessKeyID | quote }} + secretAccessKeySecretRef: + name: {{ .secretAccessKeySecretName }} + key: {{ .secretAccessKeySecretKey }} + selector: + dnsZones: + - {{ $.Values.dns01.zoneName | quote }} + {{- end }} + # HTTP-01 stays in place for every hostname outside the DNS-01 zone. cert-manager prefers + # the DNS-01 solver above, because its dnsZones selector is more specific than this one. + - http01: + ingress: + ingressClassName: {{ .Values.ingressClassName }} diff --git a/modules/kubernetes/ingress/buildingblock/templates/wildcard-certificate.yaml b/modules/kubernetes/ingress/buildingblock/templates/wildcard-certificate.yaml new file mode 100644 index 00000000..2ae5be3d --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/templates/wildcard-certificate.yaml @@ -0,0 +1,18 @@ +{{- if .Values.wildcardCertificate.enabled }} +# One certificate covers every hostname in the zone. It lives in the HAProxy namespace, which +# outlives any application namespace, so tearing an application down never takes the certificate +# with it. HAProxy serves the resulting secret as its default TLS certificate. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.wildcardCertificate.name }} + namespace: {{ .Values.wildcardCertificate.namespace }} +spec: + secretName: {{ .Values.wildcardCertificate.secretName }} + dnsNames: + - {{ printf "*.%s" .Values.dns01.zoneName | quote }} + issuerRef: + name: {{ .Values.clusterIssuer.name }} + kind: ClusterIssuer + group: cert-manager.io +{{- end }} diff --git a/modules/kubernetes/ingress/buildingblock/variables.tf b/modules/kubernetes/ingress/buildingblock/variables.tf new file mode 100644 index 00000000..2dbe5db0 --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/variables.tf @@ -0,0 +1,151 @@ +variable "cluster_endpoint" { + type = string + description = "IP address or hostname of the cluster control plane, without the https:// scheme." +} + +variable "cluster_ca_certificate" { + type = string + description = "Cluster CA certificate, base64 encoded." +} + +variable "token" { + type = string + sensitive = true + description = "Token of the service account this module runs as. It needs cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC." +} + +variable "cert_manager_version" { + type = string + default = "v1.20.0" + description = "Version of the cert-manager Helm chart. See https://github.com/cert-manager/cert-manager/releases." +} + +variable "cert_manager_namespace" { + type = string + default = "cert-manager" + description = "Namespace for cert-manager and, when DNS-01 runs through STACKIT, for the STACKIT cert-manager webhook. The webhook chart expects both in the same namespace." +} + +variable "cert_manager_extra_args" { + type = list(string) + # Cuts the retry backoff for failed certificate requests from the default 1h down to 1m, so an + # ACME order recovers quickly after a transient DNS or ingress problem. + default = ["--certificate-request-minimum-backoff-duration=1m"] + description = "Extra command line arguments for the cert-manager controller." +} + +variable "cert_manager_crds_keep" { + type = bool + default = false + description = "Keep the cert-manager CRDs when the Helm release is destroyed. Keeping them preserves existing Certificate and ClusterIssuer objects across a reinstall." +} + +variable "haproxy_version" { + type = string + default = "1.49.0" + description = "Version of the haproxytech/kubernetes-ingress Helm chart. See https://github.com/haproxytech/helm-charts/blob/main/kubernetes-ingress/Chart.yaml." +} + +variable "haproxy_namespace" { + type = string + default = "haproxy-ingress" + description = "Namespace for the HAProxy ingress controller. The wildcard certificate is created here as well, so its secret survives the teardown of any application namespace." +} + +variable "haproxy_release_name" { + type = string + default = "haproxy" + description = "Helm release name of the HAProxy ingress controller. The chart names the controller Service '-kubernetes-ingress'." +} + +variable "haproxy_replica_count" { + type = number + # Equals the chart default. It is exposed so a foundation can scale the controller without + # having to reach into the chart values. + default = 2 + description = "Number of HAProxy ingress controller replicas." +} + +variable "haproxy_service_type" { + type = string + # The chart defaults to NodePort, which gives no public address at all, so this value is + # load-bearing rather than cosmetic. + default = "LoadBalancer" + description = "Service type of the HAProxy ingress controller." +} + +variable "haproxy_service_annotations" { + type = map(string) + default = {} + description = <<-EOT + Annotations on the HAProxy controller Service. The cloud provider reads them to configure the + load balancer. Two values matter in practice: + - AKS needs `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path = "/healthz"`. + - STACKIT uses `lb.stackit.cloud/internal-lb` to keep the load balancer off the public internet. + EOT +} + +variable "haproxy_timeout" { + type = number + default = 1200 + description = "Seconds to wait for the HAProxy Helm release to become ready. The default of 20 minutes covers the time a cloud provider takes to provision the load balancer." +} + +variable "ingress_class_name" { + type = string + default = "haproxy" + description = "Name of the IngressClass the controller serves. The HTTP-01 solver of the ClusterIssuer uses the same name." +} + +variable "acme_email" { + type = string + description = "Contact address Let's Encrypt uses for expiry warnings and account recovery." +} + +variable "acme_server" { + type = string + default = "https://acme-v02.api.letsencrypt.org/directory" + description = "ACME directory URL. Point this at https://acme-staging-v02.api.letsencrypt.org/directory while you test, because the production endpoint has strict rate limits." +} + +variable "cluster_issuer_name" { + type = string + default = "letsencrypt-prod" + description = "Name of the ClusterIssuer. Application teams reference it from the cert-manager.io/cluster-issuer annotation on their Ingress." +} + +variable "acme_private_key_secret_name" { + type = string + default = "letsencrypt-prod-account-key" + description = "Name of the secret in which cert-manager stores the ACME account private key." +} + +variable "wildcard_certificate_name" { + type = string + default = "wildcard-tls" + description = "Name of the wildcard Certificate and of the secret it writes, both in haproxy_namespace. Only used when dns01 is set." +} + +variable "stackit_webhook_version" { + type = string + default = "0.4.10" + description = "Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook." +} + +variable "dns01" { + description = "Enables a wildcard certificate for zone_name via DNS-01. Set exactly one provider. Null keeps HTTP-01 per-hostname issuance." + type = object({ + zone_name = string + stackit = optional(object({ project_id = string, service_account_key = string })) + route53 = optional(object({ hosted_zone_id = string, access_key_id = string, secret_access_key = string, region = optional(string, "eu-central-1") })) + }) + default = null + sensitive = true + + validation { + condition = var.dns01 == null || ( + (try(var.dns01.stackit, null) == null ? 0 : 1) + (try(var.dns01.route53, null) == null ? 0 : 1) == 1 + ) + error_message = "Set exactly one DNS-01 provider in var.dns01: either stackit or route53." + } +} diff --git a/modules/kubernetes/ingress/buildingblock/versions.tf b/modules/kubernetes/ingress/buildingblock/versions.tf new file mode 100644 index 00000000..cc6c09d8 --- /dev/null +++ b/modules/kubernetes/ingress/buildingblock/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + helm = { + source = "hashicorp/helm" + # The helm provider takes its cluster credentials as the `kubernetes = {}` attribute + # starting with 3.0.0. Earlier versions expect a `kubernetes {}` block instead. + version = ">= 3.0.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.38" + } + } +} From 2f3bc4f0a74afd425b580937eb5022c08e74fbff Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 22:39:30 +0200 Subject: [PATCH 16/60] feat(stackit/ske): add SKE cluster building block Provisions a STACKIT Kubernetes Engine cluster and its kubeconfig, and nothing else. It replaces the three identical copies of cluster.tf that LCF, ICF and TCF each carried under platforms/ske/kubernetes/. The module lives under modules/stackit/ because that directory follows the platform the block is ordered on, not the thing it produces. modules/ske/ stays the platform whose tenants are namespaces. Over the foundation original it adds: - stackit provider >= 0.88.0 instead of ~> 0.83, which is both the hub rule and the floor for the SNA and access_scope fields - an optional network_id input for STACKIT Network Area placement, meant to come from modules/stackit/network through a BUILDING_BLOCK_OUTPUT - an optional control_plane_access_scope, unset by default so SKE creates a public control plane and no private-preview field reaches accounts that are not enabled for it - an optional dns_extension for the SKE managed ExternalDNS extension, off by default - an 11 character validation on cluster_name, the limit TCF hit when it had to shorten try-meshstack to try-mesh - node_pools and maintenance as inputs, defaulting to the foundation values The provider_config and kubeconfig outputs keep their shape so the foundations' terragrunt generate "provider" blocks keep working. A new non-sensitive kube_host output carries the API server URL on its own. Co-Authored-By: Claude Opus 5 (1M context) --- .../ske/buildingblock/APP_TEAM_README.md | 53 +++++++ modules/stackit/ske/buildingblock/README.md | 129 ++++++++++++++++++ modules/stackit/ske/buildingblock/logo.png | Bin 0 -> 1878 bytes modules/stackit/ske/buildingblock/main.tf | 46 +++++++ modules/stackit/ske/buildingblock/outputs.tf | 29 ++++ modules/stackit/ske/buildingblock/provider.tf | 8 ++ .../stackit/ske/buildingblock/variables.tf | 117 ++++++++++++++++ modules/stackit/ske/buildingblock/versions.tf | 12 ++ 8 files changed, 394 insertions(+) create mode 100644 modules/stackit/ske/buildingblock/APP_TEAM_README.md create mode 100644 modules/stackit/ske/buildingblock/README.md create mode 100644 modules/stackit/ske/buildingblock/logo.png create mode 100644 modules/stackit/ske/buildingblock/main.tf create mode 100644 modules/stackit/ske/buildingblock/outputs.tf create mode 100644 modules/stackit/ske/buildingblock/provider.tf create mode 100644 modules/stackit/ske/buildingblock/variables.tf create mode 100644 modules/stackit/ske/buildingblock/versions.tf diff --git a/modules/stackit/ske/buildingblock/APP_TEAM_README.md b/modules/stackit/ske/buildingblock/APP_TEAM_README.md new file mode 100644 index 00000000..4afd621a --- /dev/null +++ b/modules/stackit/ske/buildingblock/APP_TEAM_README.md @@ -0,0 +1,53 @@ +This building block gives you a STACKIT Kubernetes Engine (SKE) cluster with one autoscaling node +pool, a nightly maintenance window and a kubeconfig you can use right away. The platform team runs +the control plane, the node images and the Kubernetes version updates, and you run the workloads on +top of it. + +## 🎯 When to use it + +Use this building block when you: + +- need a full Kubernetes cluster of your own rather than a namespace on a shared cluster +- want your workloads to stay on European infrastructure operated by STACKIT +- want STACKIT to handle control plane operation, node image updates and Kubernetes version updates +- plan to place the cluster inside a STACKIT Network Area so the control plane stays off the public internet + +## πŸ’‘ Usage examples + +**Example 1: A team platform for several environments** +Your team runs a set of services that need their own cluster-wide resources, such as custom resource +definitions and cluster-scoped operators. You order the cluster, take the kubeconfig from the outputs +and deploy your namespaces, operators and workloads into it. + +**Example 2: A cluster behind a STACKIT Network Area** +Your workloads process data that must not be reachable from the public internet. You order the cluster +with a network id from your STACKIT network and set the control plane access scope to `SNA`, so the +Kubernetes API server answers only inside your network area. + +## πŸ”‘ Getting the kubeconfig + +The building block exposes the kubeconfig as a sensitive output. Write it to a file and point +`kubectl` at it: + +```bash +tofu output -raw kubeconfig > kubeconfig +kubectl --kubeconfig kubeconfig get nodes +``` + +The kubeconfig expires after 180 days by default. Terraform refreshes it on the next run once it +passes half of its lifetime, so keep applying the building block regularly. + +## πŸ“Š Shared Responsibility + +| Responsibility | Platform Team | Application Team | +|---|:---:|:---:| +| Provision the SKE cluster and its node pool | βœ… | ❌ | +| Operate the Kubernetes control plane | βœ… | ❌ | +| Apply Kubernetes and machine image updates in the maintenance window | βœ… | ❌ | +| Provide the network and decide the control plane access scope | βœ… | ❌ | +| Rotate the kubeconfig before it expires | βœ… | ❌ | +| Deploy and operate workloads on the cluster | ❌ | βœ… | +| Set resource requests and limits for workloads | ❌ | βœ… | +| Manage namespaces and in-cluster RBAC | ❌ | βœ… | +| Store the kubeconfig somewhere safe | ❌ | βœ… | +| Monitor application health and logs | ❌ | βœ… | diff --git a/modules/stackit/ske/buildingblock/README.md b/modules/stackit/ske/buildingblock/README.md new file mode 100644 index 00000000..a36652a6 --- /dev/null +++ b/modules/stackit/ske/buildingblock/README.md @@ -0,0 +1,129 @@ +--- +name: STACKIT Kubernetes Engine (SKE) Cluster +supportedPlatforms: + - stackit +description: Provisions a STACKIT Kubernetes Engine cluster with one node pool, a maintenance window and a kubeconfig. +--- + +# SKE Cluster Building Block + +This module provisions a [STACKIT Kubernetes Engine](https://docs.stackit.cloud/products/runtime/kubernetes-engine/) +cluster and the kubeconfig that gives access to it. It provisions nothing else β€” no addons, no +meshStack registration, no DNS records. Compose it with other modules to build a full platform: + +- `modules/kubernetes/platform` registers the cluster as a meshStack platform whose tenants are namespaces. +- `modules/kubernetes/ingress` installs an ingress controller and certificate management. +- `modules/stackit/network` supplies the network for STACKIT Network Area (SNA) placement. + +The module lives under `modules/stackit/` because that directory follows the platform the block is +**ordered on**, not the thing it produces. `modules/ske/` stays the platform whose tenants are +namespaces on an existing cluster. + +## Why this module lives in the hub + +The `likvid-cloudfoundation`, `internal-cloudfoundation` and `trial-cloudfoundation` foundations each +carried their own copy of this cluster definition. The copies were identical apart from the provider +block. This module replaces all three. + +## Outputs the foundations rely on + +The module keeps two outputs the foundations already consume through Terragrunt: + +- `provider_config` β€” a ready-made object with `host`, `cluster_ca_certificate`, `client_certificate` + and `client_key`, which the foundations feed into a generated `kubernetes` and `helm` provider block. +- `kubeconfig` β€” the decoded kubeconfig. The foundations read it as a fallback when `provider_config` + is missing, so it stays part of the contract. + +A third output, `kube_host`, exposes the API server URL on its own and is not sensitive, so a +composition can pass it to `modules/kubernetes/platform` without tainting the whole value. + +## STACKIT Network Area placement + +Set `network_id` to place the cluster on a network you created yourself, for example with +`modules/stackit/network`. In meshStack, wire this through a `BUILDING_BLOCK_OUTPUT` input β€” a network +id is not a secret, so nothing on the meshStack side blocks that. Leave `network_id` unset and SKE +places the cluster on a network of its own. + +## Control plane access scope + +`control_plane_access_scope` decides whether the Kubernetes control plane is reachable from the +public internet (`PUBLIC`) or only from inside a STACKIT Network Area (`SNA`). Three constraints apply +and you should read all three before you set it: + +1. **The field is immutable.** STACKIT fixes the access scope when it creates the cluster. Changing the + value later forces you to replace the cluster. +2. **The field is feature-flagged, not generally available.** STACKIT rejects the request unless your + account is enabled for it, and enabling it takes a support ticket per organization or per project. + Because of this, the module sends no `control_plane` block at all while the variable is unset. +3. **The field conflicts with the ACL extension.** A cluster cannot use `access_scope` and the SKE ACL + extension (`extensions.acl`) at the same time. This module does not expose the ACL extension, which + keeps the two apart. + +Leaving `control_plane_access_scope` unset gives you a public control plane, which is the default for +this module. + +## Managed ExternalDNS extension + +`dns_extension` turns on the SKE managed ExternalDNS extension. It is off by default. `zones` is the +domain filter ExternalDNS applies: + +```hcl +dns_extension = { + enabled = true + zones = ["apps.example.runs.onstackit.cloud"] +} +``` + +A wildcard in the `zones` filter is not written as `*` into the DNS records ExternalDNS creates. +STACKIT rewrites it to the literal label `x-stackit-dns-wildcard`, so a record for +`*.apps.example.com` appears as `x-stackit-dns-wildcard.apps.example.com` in the STACKIT DNS zone. +Keep that in mind when you look for a record you expect to find under `*`. + +## Cluster name length + +STACKIT limits SKE cluster names to 11 characters and the module validates that. The trial foundation +ran into this limit and had to shorten `try-meshstack` to `try-mesh`. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [stackit](#requirement\_stackit) | >= 0.88.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_ske_cluster.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/ske_cluster) | resource | +| [stackit_ske_kubeconfig.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/ske_kubeconfig) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_name](#input\_cluster\_name) | Name of the SKE cluster. STACKIT limits SKE cluster names to 11 characters. | `string` | n/a | yes | +| [control\_plane\_access\_scope](#input\_control\_plane\_access\_scope) | Access scope of the control plane, either `PUBLIC` or `SNA`. Leave unset to get a public control plane, which is what SKE creates by default. The field is immutable after creation, it is feature-flagged per organization or project, and it cannot be combined with the ACL extension. | `string` | `null` | no | +| [dns\_extension](#input\_dns\_extension) | SKE managed ExternalDNS extension. Leave unset to keep the extension off. `zones` is the domain filter ExternalDNS applies, for example `["apps.example.runs.onstackit.cloud"]`. | 
object({
    enabled     = optional(bool, true)
    zones       = optional(list(string))
    gateway_api = optional(bool)
  })
| `null` | no | +| [kubeconfig\_expiration\_seconds](#input\_kubeconfig\_expiration\_seconds) | Lifetime of the generated kubeconfig in seconds. Terraform refreshes the kubeconfig once it reaches half of this lifetime. | `number` | `15552000` | no | +| [maintenance](#input\_maintenance) | Maintenance window in which SKE applies Kubernetes and machine image updates. | 
object({
    enable_kubernetes_version_updates    = optional(bool, true)
    enable_machine_image_version_updates = optional(bool, true)
    start                                = optional(string, "01:00:00Z")
    end                                  = optional(string, "02:00:00Z")
  })
| `{}` | no | +| [network\_id](#input\_network\_id) | UUID of the STACKIT Network Area (SNA) network the cluster is deployed into. Feed this from the `network_id` output of `modules/stackit/network`. Leave unset to let SKE place the cluster on its own network. | `string` | `null` | no | +| [node\_pools](#input\_node\_pools) | Node pools of the cluster. The default is a single autoscaling pool of general instances, which is what the meshcloud foundations run today. | 
list(object({
    name                    = string
    machine_type            = string
    minimum                 = number
    maximum                 = number
    availability_zones      = list(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    allow_system_components = optional(bool)
    labels                  = optional(map(string))
    os_name                 = optional(string)
    os_version_min          = optional(string)
    volume_type             = optional(string)
    volume_size             = optional(number)
  }))
| 
[
  {
    "availability_zones": [
      "eu01-1"
    ],
    "machine_type": "g2i.2",
    "max_surge": 1,
    "maximum": 3,
    "minimum": 1,
    "name": "pool-1"
  }
]
| no | +| [service\_account\_email](#input\_service\_account\_email) | Email of the STACKIT service account the provider authenticates as via workload identity federation. Leave unset when the caller supplies its own provider configuration. | `string` | `null` | no | +| [stackit\_project\_id](#input\_stackit\_project\_id) | STACKIT project UUID that holds the SKE cluster. | `string` | n/a | yes | +| [stackit\_region](#input\_stackit\_region) | STACKIT region the provider talks to. Ignored when the caller supplies its own provider configuration. | `string` | `"eu01"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_name](#output\_cluster\_name) | Name of the SKE cluster. | +| [kube\_host](#output\_kube\_host) | URL of the Kubernetes API server. Feed this into the `kube_host` input of `modules/kubernetes/platform`. | +| [kubeconfig](#output\_kubeconfig) | Raw kubeconfig content for cluster access. | +| [provider\_config](#output\_provider\_config) | Kubernetes provider config values derived from the kubeconfig, for wiring a kubernetes or helm provider. | + diff --git a/modules/stackit/ske/buildingblock/logo.png b/modules/stackit/ske/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..225391782a085772b79e9ce3d491e62e933a940b GIT binary patch literal 1878 zcmZ8i2~?6<6#mQHCCjnK(l(cza%3*YfuRsEa0DL$4O$-xtp}b4MMWVo zfeMjAp^y-y03^Mv1$zUw z4poaVh9R*ztJC4?ulWj;;bZ0`Y)R7wF>gasHDB$$xQupA>x&glpWun7U!X z6yQ-nUiT1VI|15|w@CB~(tsaHH7}U30rKe!DYTvXWx~+sH{JTfGnX5D{Cm<)ZwzVm z26H|C-IKMKgpD=n^)GKu?72mac~WN-#!cL^CfN1oUA{|~ESk$PnQdWa!`#fD<*_~1 z;aOXD_4(g#yq0BUJiPGWeNNtJ$?>Kl(bf4s1{zPR>_|N_eSv2ZAv!MJ*)eL0U2-HZ z!g7_w>Li>k+$@!KgpE?hYsxj z<92SQebBlJK|$~0{N5)kPC(G4$#|bN8+WwzswtEWYb^Dj&vlsKow3)Xk$S+OZX$Jx zsZoKzX!<0dT5q@EXZ-%^k!So)>1bTu@R?<7QDe`XZExtwk9^tfx!YIzY!4ZFbstJw zImi!q?+fSOXT={=QI0D-HN1O^EwtwD*ac^5^IR$%OX5a8aJ9}BlAB)e96#2GbycBh zK+h@er#Q#-Y9Dx{q^i*W_3%9Vp-%!ePFTy>GEgYN_sBFu0a?8`5ksGGm$Vx`=%v{YowgOH%u7%C?e2AtBelE91ma+(vh#i~)A{CG zvN*-n7%P#g?+9ygkDrHRIEdKyyZ1Y*Qi|2H>sfADs*X%WB&J+zYoU#H$70Xc!UK&F z2J$50F;zliYql+K$wpXOv_tr`eqr5`DsFJ3Wy8RqXD;s)QtDls@0@M8V^tRaywr`n zO6KG^0VliqxYaQ>@5QOX>1A%6rK{2OHvKZ`FUs=hql1h?uE{o51{E5(^*oW9>bl~G)8J69EY7}oU3@HN8@=x^hCf3R zYPZM?mRCoYGUL6o<$4Hv_9jM)2~QkTQw;Nz9P6PZ-ZHl3A`C3gEoR31Quz92G8I=9 zr@XCbc`IN7pu`$swO;rQl&z0CO58kO5{5$>+RK>new09*Y0y3u*O8-C+1|CZ^$)3Z zdHOQKRGszL@*cu+ufB$mh{`?&VuEmqva1%ni&s}yaSI{<&rF@ zpIo7N+tl5oBJ-zpBz?8oD*xc;%@IF)aP8{_muwlkU&ax$DqSc(?0z{%Y*tqumQV?Q zF2GXYH0i>ad*@v!f!-^k4iHmgX$A$ABX?wCC+B5A&2q zf~v9-C8muV?wkZNhQ{r7U+b*Dgy7xHHAi#bMAaFH5#Pl1ls`I1qPza&yut}6k*-ZB zWtDvvHK~iJb`_=SB4q*zW{YW4x61YRYodHB&JjStC3PR5-ub`Y^Tq0-j{$_MVw#8m zBR&l7j}mptdjT|#V9^uR5o#oQ@f=t~|Nuh`oom fvBSwdOZPyP@64?#ZB$msKSI1O!RLl|#NK}ZQo#tq literal 0 HcmV?d00001 diff --git a/modules/stackit/ske/buildingblock/main.tf b/modules/stackit/ske/buildingblock/main.tf new file mode 100644 index 00000000..9c177ec3 --- /dev/null +++ b/modules/stackit/ske/buildingblock/main.tf @@ -0,0 +1,46 @@ +resource "stackit_ske_cluster" "this" { + project_id = var.stackit_project_id + name = var.cluster_name + + node_pools = var.node_pools + + maintenance = { + enable_kubernetes_version_updates = var.maintenance.enable_kubernetes_version_updates + enable_machine_image_version_updates = var.maintenance.enable_machine_image_version_updates + start = var.maintenance.start + end = var.maintenance.end + } + + network = local.network + extensions = local.extensions +} + +resource "stackit_ske_kubeconfig" "this" { + project_id = var.stackit_project_id + cluster_name = stackit_ske_cluster.this.name + expiration = var.kubeconfig_expiration_seconds + refresh = true +} + +locals { + # `network` stays unset unless the caller asks for something, because SKE rejects a + # `control_plane` block from accounts that are not enabled for the feature. + network = var.network_id == null && var.control_plane_access_scope == null ? null : { + id = var.network_id + control_plane = var.control_plane_access_scope == null ? null : { + access_scope = var.control_plane_access_scope + } + } + + extensions = var.dns_extension == null ? null : { + dns = { + enabled = var.dns_extension.enabled + zones = var.dns_extension.zones + gateway_api = var.dns_extension.gateway_api + } + } + + kubeconfig = yamldecode(stackit_ske_kubeconfig.this.kube_config) + kubeconfig_cluster = one(local.kubeconfig.clusters).cluster + kubeconfig_admin_user = one(local.kubeconfig.users).user +} diff --git a/modules/stackit/ske/buildingblock/outputs.tf b/modules/stackit/ske/buildingblock/outputs.tf new file mode 100644 index 00000000..ca74cf23 --- /dev/null +++ b/modules/stackit/ske/buildingblock/outputs.tf @@ -0,0 +1,29 @@ +output "cluster_name" { + description = "Name of the SKE cluster." + value = stackit_ske_cluster.this.name +} + +output "kube_host" { + description = "URL of the Kubernetes API server. Feed this into the `kube_host` input of `modules/kubernetes/platform`." + # The whole kubeconfig is sensitive because it carries the client certificate and key. The API + # server URL is not a secret, so it is declassified here and a composition can pass it on as a + # plain string. + value = nonsensitive(local.kubeconfig_cluster.server) +} + +output "kubeconfig" { + description = "Raw kubeconfig content for cluster access." + value = local.kubeconfig + sensitive = true +} + +output "provider_config" { + description = "Kubernetes provider config values derived from the kubeconfig, for wiring a kubernetes or helm provider." + value = { + host = local.kubeconfig_cluster.server + cluster_ca_certificate = base64decode(local.kubeconfig_cluster["certificate-authority-data"]) + client_certificate = base64decode(local.kubeconfig_admin_user["client-certificate-data"]) + client_key = base64decode(local.kubeconfig_admin_user["client-key-data"]) + } + sensitive = true +} diff --git a/modules/stackit/ske/buildingblock/provider.tf b/modules/stackit/ske/buildingblock/provider.tf new file mode 100644 index 00000000..adac1a15 --- /dev/null +++ b/modules/stackit/ske/buildingblock/provider.tf @@ -0,0 +1,8 @@ +# Callers that drive this module from Terragrunt usually replace this file with a generated +# `provider.tf` of their own. In that case `service_account_email` and `stackit_region` stay +# unset and the generated block carries the credentials instead. +provider "stackit" { + default_region = var.stackit_region + service_account_email = var.service_account_email + use_oidc = true +} diff --git a/modules/stackit/ske/buildingblock/variables.tf b/modules/stackit/ske/buildingblock/variables.tf new file mode 100644 index 00000000..c46ad502 --- /dev/null +++ b/modules/stackit/ske/buildingblock/variables.tf @@ -0,0 +1,117 @@ +variable "stackit_project_id" { + type = string + nullable = false + description = "STACKIT project UUID that holds the SKE cluster." +} + +variable "cluster_name" { + type = string + nullable = false + description = "Name of the SKE cluster. STACKIT limits SKE cluster names to 11 characters." + + validation { + condition = length(var.cluster_name) >= 1 && length(var.cluster_name) <= 11 + error_message = "SKE cluster names are limited to 11 characters." + } + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9-]*[a-z0-9])?$", var.cluster_name)) + error_message = "SKE cluster names may contain lowercase letters, digits and hyphens, and must start and end with a letter or a digit." + } +} + +variable "stackit_region" { + type = string + nullable = true + default = "eu01" + description = "STACKIT region the provider talks to. Ignored when the caller supplies its own provider configuration." +} + +variable "service_account_email" { + type = string + nullable = true + default = null + description = "Email of the STACKIT service account the provider authenticates as via workload identity federation. Leave unset when the caller supplies its own provider configuration." +} + +variable "node_pools" { + type = list(object({ + name = string + machine_type = string + minimum = number + maximum = number + availability_zones = list(string) + max_surge = optional(number) + max_unavailable = optional(number) + allow_system_components = optional(bool) + labels = optional(map(string)) + os_name = optional(string) + os_version_min = optional(string) + volume_type = optional(string) + volume_size = optional(number) + })) + nullable = false + + default = [ + { + name = "pool-1" + machine_type = "g2i.2" # general instances + minimum = 1 + maximum = 3 + availability_zones = ["eu01-1"] + max_surge = 1 + } + ] + + description = "Node pools of the cluster. The default is a single autoscaling pool of general instances, which is what the meshcloud foundations run today." +} + +variable "maintenance" { + type = object({ + enable_kubernetes_version_updates = optional(bool, true) + enable_machine_image_version_updates = optional(bool, true) + start = optional(string, "01:00:00Z") + end = optional(string, "02:00:00Z") + }) + nullable = false + default = {} + + description = "Maintenance window in which SKE applies Kubernetes and machine image updates." +} + +variable "network_id" { + type = string + nullable = true + default = null + description = "UUID of the STACKIT Network Area (SNA) network the cluster is deployed into. Feed this from the `network_id` output of `modules/stackit/network`. Leave unset to let SKE place the cluster on its own network." +} + +variable "control_plane_access_scope" { + type = string + nullable = true + default = null + description = "Access scope of the control plane, either `PUBLIC` or `SNA`. Leave unset to get a public control plane, which is what SKE creates by default. The field is immutable after creation, it is feature-flagged per organization or project, and it cannot be combined with the ACL extension." + + validation { + condition = var.control_plane_access_scope == null ? true : contains(["PUBLIC", "SNA"], var.control_plane_access_scope) + error_message = "control_plane_access_scope must be either PUBLIC or SNA." + } +} + +variable "dns_extension" { + type = object({ + enabled = optional(bool, true) + zones = optional(list(string)) + gateway_api = optional(bool) + }) + nullable = true + default = null + description = "SKE managed ExternalDNS extension. Leave unset to keep the extension off. `zones` is the domain filter ExternalDNS applies, for example `[\"apps.example.runs.onstackit.cloud\"]`." +} + +variable "kubeconfig_expiration_seconds" { + type = number + nullable = false + default = 15552000 # 180 days + description = "Lifetime of the generated kubeconfig in seconds. Terraform refreshes the kubeconfig once it reaches half of this lifetime." +} diff --git a/modules/stackit/ske/buildingblock/versions.tf b/modules/stackit/ske/buildingblock/versions.tf new file mode 100644 index 00000000..0a37589b --- /dev/null +++ b/modules/stackit/ske/buildingblock/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + # 0.88.0 is the first release that carries `network.id` for STACKIT Network Area + # placement and `network.control_plane.access_scope` for the control plane scope. + version = ">= 0.88.0" + } + } +} From 06f77bd347c103e60cc73af5e09133e3f1523165 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 22:39:47 +0200 Subject: [PATCH 17/60] feat(kubernetes/platform): add Kubernetes platform registration building block Registers a Kubernetes cluster as a meshStack platform of type kubernetes, adds one namespace landing zone per environment, and creates the in-cluster replicator and metering service accounts meshStack authenticates with. Nothing in the module names a cloud, so it serves SKE today and any conformant cluster later. It is built from the two SKE-side copies in LCF and TCF plus the hand-rolled service account in modules/ske/ske-starterkit/e2e. Every value the three disagreed on became a variable with the LCF value as the default: the quota definitions, the landing zones and their quotas and tags, the platform name, display name and description, the location, the namespace name pattern and the metering toggle. The module hand-rolls the service accounts instead of sourcing terraform-kubernetes-meshplatform v0.2.0, which keeps the hub self-contained. The resource names match that module, so an existing deployment can be moved over without renaming. The AKS copies cannot share this module. meshStack models them with spec.config.aks, an Entra service principal read from data.meshstack_integrations and six extra required fields, so AKS needs its own registration module. Co-Authored-By: Claude Opus 5 (1M context) --- .../platform/buildingblock/APP_TEAM_README.md | 44 +++ .../platform/buildingblock/README.md | 126 ++++++++ .../platform/buildingblock/logo.png | Bin 0 -> 10107 bytes .../kubernetes/platform/buildingblock/main.tf | 204 +++++++++++++ .../buildingblock/meshstack_landingzone.tf | 42 +++ .../buildingblock/meshstack_platform.tf | 62 ++++ .../platform/buildingblock/outputs.tf | 34 +++ .../platform/buildingblock/provider.tf | 9 + .../platform/buildingblock/variables.tf | 269 ++++++++++++++++++ .../platform/buildingblock/versions.tf | 15 + 10 files changed, 805 insertions(+) create mode 100644 modules/kubernetes/platform/buildingblock/APP_TEAM_README.md create mode 100644 modules/kubernetes/platform/buildingblock/README.md create mode 100644 modules/kubernetes/platform/buildingblock/logo.png create mode 100644 modules/kubernetes/platform/buildingblock/main.tf create mode 100644 modules/kubernetes/platform/buildingblock/meshstack_landingzone.tf create mode 100644 modules/kubernetes/platform/buildingblock/meshstack_platform.tf create mode 100644 modules/kubernetes/platform/buildingblock/outputs.tf create mode 100644 modules/kubernetes/platform/buildingblock/provider.tf create mode 100644 modules/kubernetes/platform/buildingblock/variables.tf create mode 100644 modules/kubernetes/platform/buildingblock/versions.tf diff --git a/modules/kubernetes/platform/buildingblock/APP_TEAM_README.md b/modules/kubernetes/platform/buildingblock/APP_TEAM_README.md new file mode 100644 index 00000000..79914d8f --- /dev/null +++ b/modules/kubernetes/platform/buildingblock/APP_TEAM_README.md @@ -0,0 +1,44 @@ +This building block connects a Kubernetes cluster to meshStack, so that teams can order a namespace +on it from the marketplace. It creates the service accounts meshStack authenticates with inside the +cluster, registers the cluster as a platform, and adds one landing zone per environment with its own +quotas. After it runs, meshStack replicates every tenant of the platform into a namespace on the +cluster and keeps the role bindings in sync with the project roles. + +## 🎯 When to use it + +Use this building block when you: + +- run a Kubernetes cluster and want to hand out namespaces on it through meshStack instead of by hand +- want project roles in meshStack to drive `admin`, `edit` and `view` access inside the namespace +- want meshStack to enforce CPU, memory and storage quotas per namespace +- want usage data from the cluster to reach meshMetering for chargeback + +Do not use it for AKS. meshStack models AKS namespace platforms with a different configuration and an +Entra service principal, so AKS needs its own registration module. + +## πŸ’‘ Usage examples + +**Example 1: Opening a new cluster to the marketplace** +Your team has just finished a STACKIT Kubernetes Engine cluster and wants application teams to be able +to order namespaces on it. You order this building block with the cluster's API server URL and its +credentials. Application teams then see a `dev` and a `prod` landing zone in the marketplace and can +order a namespace on either. + +**Example 2: Tightening the quotas on a small cluster** +Your cluster is smaller than the defaults assume, so you want half the CPU and memory per namespace. +You order the building block with your own `quota_definitions` and `landing_zones` values, and +meshStack rejects any tenant request that goes over the new limits. + +## πŸ“Š Shared Responsibility + +| Responsibility | Platform Team | Application Team | +|---|:---:|:---:| +| Provide and operate the Kubernetes cluster | βœ… | ❌ | +| Create the replicator and metering service accounts in the cluster | βœ… | ❌ | +| Register the platform and its landing zones in meshStack | βœ… | ❌ | +| Choose the quota limits and the auto-approval thresholds | βœ… | ❌ | +| Rotate the cluster credentials the registration uses | βœ… | ❌ | +| Order a namespace on one of the landing zones | ❌ | βœ… | +| Deploy and operate workloads inside the namespace | ❌ | βœ… | +| Request a quota increase when a workload outgrows the landing zone | ❌ | βœ… | +| Monitor application health and logs | ❌ | βœ… | diff --git a/modules/kubernetes/platform/buildingblock/README.md b/modules/kubernetes/platform/buildingblock/README.md new file mode 100644 index 00000000..8e5e8a41 --- /dev/null +++ b/modules/kubernetes/platform/buildingblock/README.md @@ -0,0 +1,126 @@ +--- +name: Kubernetes Platform Registration +supportedPlatforms: + - kubernetes +description: Registers a Kubernetes cluster as a meshStack platform of type kubernetes, with its namespace landing zones and the in-cluster service accounts meshStack authenticates with. +# The module creates its identities inside the target cluster and receives the cluster credentials +# as inputs, so there is no cloud-side setup to perform ahead of time. +requiresBackplane: false +--- + +# Kubernetes Platform Registration Building Block + +This module registers a Kubernetes cluster as a meshStack platform whose tenants are namespaces. It +creates three things: + +1. The in-cluster identities meshStack authenticates with β€” a replicator service account that creates + namespaces, resource quotas and role bindings, and a metering service account that reads pods and + persistent volume claims. +2. A `meshstack_platform` of type `kubernetes` that points at the cluster's API server. +3. One `meshstack_landingzone` per environment, each with its own quotas and tags. + +Nothing in the module names a cloud provider. It serves STACKIT Kubernetes Engine today and any +conformant cluster later, as long as the cluster accepts service account tokens for authentication. + +## Why this module cannot serve AKS + +meshStack models AKS namespace platforms differently, and the difference is a modelling fact rather +than a design choice: + +- AKS uses `spec.config.aks`, not `spec.config.kubernetes`, and its landing zones use + `platform_properties.aks`. +- AKS authenticates with an Entra service principal through workload identity federation, read from + `data.meshstack_integrations`, instead of an in-cluster service account token. +- The AKS config carries fields that have no counterpart here: `group_name_pattern`, + `user_lookup_strategy`, `send_azure_invitation_mail`, `redirect_url`, `aks_subscription_id`, + `aks_cluster_name`, `aks_resource_group` and the Entra tenant. + +An AKS platform registration therefore needs its own module. + +## What the module replaces + +Five copies of this configuration existed before, three of them on the SKE side. This module is built +from those three and turns every value they disagreed on into a variable. The defaults are the +`likvid-cloudfoundation` values. + +## Where the credentials come from + +The module needs cluster-admin credentials to create the service accounts and the cluster roles. +Supply them through `kube_host`, `cluster_ca_certificate`, `client_certificate` and `client_key`, for +example from the `provider_config` output of `modules/stackit/ske`. Callers that generate their own +`provider "kubernetes"` block β€” Terragrunt does this β€” can leave the three credential variables unset +and pass only `kube_host`, which meshStack also stores as the platform endpoint. + +## Running more than one registration on one cluster + +The in-cluster resource names are fixed (`meshfed-service`, `meshfed-metering`) so that an existing +deployment of the `terraform-kubernetes-meshplatform` module can be moved into this one without +renaming anything. Set `resource_name_suffix` when a single cluster carries more than one meshStack +platform registration, so the service accounts and cluster roles do not collide. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.38.0 | +| [meshstack](#requirement\_meshstack) | >= 0.20.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_cluster_role.metering](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | +| [kubernetes_cluster_role.replicator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | +| [kubernetes_cluster_role_binding.metering](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_cluster_role_binding.replicator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | +| [kubernetes_namespace.meshcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_secret.metering](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | +| [kubernetes_secret.replicator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource | +| [kubernetes_service_account.metering](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [kubernetes_service_account.replicator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [meshstack_landingzone.this](https://registry.terraform.io/providers/meshcloud/meshstack/latest/docs/resources/landingzone) | resource | +| [meshstack_platform.this](https://registry.terraform.io/providers/meshcloud/meshstack/latest/docs/resources/platform) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [client\_certificate](#input\_client\_certificate) | PEM encoded client certificate the module authenticates with while it creates the in-cluster service accounts. Leave unset when the caller supplies its own provider configuration. | `string` | `null` | no | +| [client\_key](#input\_client\_key) | PEM encoded client key that belongs to `client_certificate`. Leave unset when the caller supplies its own provider configuration. | `string` | `null` | no | +| [cluster\_ca\_certificate](#input\_cluster\_ca\_certificate) | PEM encoded CA certificate of the cluster. Leave unset when the caller supplies its own provider configuration. | `string` | `null` | no | +| [disable\_ssl\_validation](#input\_disable\_ssl\_validation) | Skip SSL validation when meshStack calls the Kubernetes API server. SKE clusters serve a certificate that meshStack does not trust by default, which is why this is on. | `bool` | `true` | no | +| [documentation\_url](#input\_documentation\_url) | Link to the platform documentation shown in meshPanel. | `string` | `""` | no | +| [kube\_host](#input\_kube\_host) | URL of the Kubernetes API server, for example `https://k8s.example.com:6443`. meshStack calls this URL to replicate tenants and to collect metering data. | `string` | n/a | yes | +| [landing\_zones](#input\_landing\_zones) | Landing zones to create for the platform, keyed by environment. The key becomes the suffix of the landing zone identifier, for example `dev` gives `ske-namespace-dev`. | 
map(object({
    display_name = string
    description  = string
    info_link    = optional(string, "")
    tags         = optional(map(list(string)), {})
    quotas       = optional(list(object({ key = string, value = number })), [])
  }))
| 
{
  "dev": {
    "description": "Landing zone for development workloads.",
    "display_name": "SKE Kubernetes Namespace – Development",
    "quotas": [
      {
        "key": "limits.cpu",
        "value": 500
      },
      {
        "key": "requests.cpu",
        "value": 250
      },
      {
        "key": "limits.memory",
        "value": 512
      },
      {
        "key": "requests.memory",
        "value": 256
      },
      {
        "key": "requests.storage",
        "value": 1
      },
      {
        "key": "persistentvolumeclaims",
        "value": 2
      }
    ],
    "tags": {
      "LandingZoneFamily": [
        "cloud-native"
      ],
      "confidentiality": [
        "internal"
      ],
      "environment": [
        "dev"
      ]
    }
  },
  "prod": {
    "description": "Landing zone for production workloads.",
    "display_name": "SKE Kubernetes Namespace – Production",
    "quotas": [
      {
        "key": "limits.cpu",
        "value": 1000
      },
      {
        "key": "requests.cpu",
        "value": 500
      },
      {
        "key": "limits.memory",
        "value": 1024
      },
      {
        "key": "requests.memory",
        "value": 512
      },
      {
        "key": "requests.storage",
        "value": 2
      },
      {
        "key": "persistentvolumeclaims",
        "value": 4
      }
    ],
    "tags": {
      "LandingZoneFamily": [
        "cloud-native"
      ],
      "confidentiality": [
        "public"
      ],
      "environment": [
        "prod"
      ]
    }
  }
}
| no | +| [location\_identifier](#input\_location\_identifier) | Identifier of the meshStack location the platform is registered in. | `string` | n/a | yes | +| [metering\_additional\_rules](#input\_metering\_additional\_rules) | Extra RBAC rules added to the metering cluster role. | 
list(object({
    api_groups        = list(string)
    resources         = list(string)
    verbs             = list(string)
    resource_names    = optional(list(string))
    non_resource_urls = optional(list(string))
  }))
| `[]` | no | +| [metering\_enabled](#input\_metering\_enabled) | Create the metering service account and register metering on the platform. Turn this off when meshStack should not collect usage data from the cluster. | `bool` | `true` | no | +| [metering\_processing](#input\_metering\_processing) | How long meshMetering keeps timelines and raw data. Only used when `metering_enabled` is true. | 
object({
    compact_timelines_after_days = optional(number, 30)
    delete_raw_data_after_days   = optional(number, 65)
  })
| `{}` | no | +| [namespace\_name\_pattern](#input\_namespace\_name\_pattern) | Pattern meshStack uses to name the namespace it creates for a tenant. | `string` | `"#{workspaceIdentifier}-#{projectIdentifier}"` | no | +| [owning\_workspace\_identifier](#input\_owning\_workspace\_identifier) | Identifier of the meshStack workspace that owns the platform and its landing zones. | `string` | n/a | yes | +| [platform\_description](#input\_platform\_description) | Description of the platform as users see it in meshPanel. | `string` | `"Provides a kubernetes namespace on STACKIT Kubernetes Engine (SKE)."` | no | +| [platform\_display\_name](#input\_platform\_display\_name) | Name of the platform as users see it in meshPanel. | `string` | `"Kubernetes namespace on SKE"` | no | +| [platform\_name](#input\_platform\_name) | meshStack platform identifier. The landing zones derive their names from it, for example `ske-namespace-dev`. | `string` | `"ske-namespace"` | no | +| [quota\_definitions](#input\_quota\_definitions) | Quota keys a tenant can request on this platform, with the upper bound and the threshold below which meshStack approves a request automatically. | 
list(object({
    quota_key               = string
    label                   = string
    description             = string
    unit                    = string
    min_value               = number
    max_value               = number
    auto_approval_threshold = number
  }))
| 
[
  {
    "auto_approval_threshold": 1000,
    "description": "The sum of CPU limits across all pods in a non-terminal state cannot exceed this value.",
    "label": "CPU limit",
    "max_value": 1000,
    "min_value": 0,
    "quota_key": "limits.cpu",
    "unit": "m"
  },
  {
    "auto_approval_threshold": 500,
    "description": "The sum of CPU requests across all pods in a non-terminal state cannot exceed this value.",
    "label": "CPU requests",
    "max_value": 1000,
    "min_value": 0,
    "quota_key": "requests.cpu",
    "unit": "m"
  },
  {
    "auto_approval_threshold": 1024,
    "description": "The sum of memory limits across all pods in a non-terminal state cannot exceed this value.",
    "label": "Memory limit",
    "max_value": 1024,
    "min_value": 0,
    "quota_key": "limits.memory",
    "unit": "Mi"
  },
  {
    "auto_approval_threshold": 512,
    "description": "The sum of memory requests across all pods in a non-terminal state cannot exceed this value.",
    "label": "Memory requests",
    "max_value": 1024,
    "min_value": 0,
    "quota_key": "requests.memory",
    "unit": "Mi"
  },
  {
    "auto_approval_threshold": 2,
    "description": "Across all persistent volume claims, the sum of storage requests cannot exceed this value.",
    "label": "Total Storage Requests",
    "max_value": 5,
    "min_value": 0,
    "quota_key": "requests.storage",
    "unit": "Gi"
  },
  {
    "auto_approval_threshold": 2,
    "description": "The total number of PersistentVolumeClaims that can exist in the namespace.",
    "label": "Persistent Volume Claims",
    "max_value": 4,
    "min_value": 0,
    "quota_key": "persistentvolumeclaims",
    "unit": ""
  }
]
| no | +| [replicator\_additional\_rules](#input\_replicator\_additional\_rules) | Extra RBAC rules added to the replicator cluster role. | 
list(object({
    api_groups        = list(string)
    resources         = list(string)
    verbs             = list(string)
    resource_names    = optional(list(string))
    non_resource_urls = optional(list(string))
  }))
| `[]` | no | +| [resource\_name\_suffix](#input\_resource\_name\_suffix) | Suffix appended to the in-cluster resource names. Set it when one cluster carries more than one meshStack platform registration, so the service accounts and cluster roles do not collide. | `string` | `""` | no | +| [service\_account\_namespace](#input\_service\_account\_namespace) | Namespace that holds the replicator and metering service accounts. | `string` | `"meshcloud"` | no | +| [support\_url](#input\_support\_url) | Link to the support channel shown in meshPanel. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [landing\_zone\_identifiers](#output\_landing\_zone\_identifiers) | meshStack landing zone identifiers keyed by environment. | +| [landing\_zone\_refs](#output\_landing\_zone\_refs) | meshStack landing zone references keyed by environment, for use in building block compositions. | +| [metering\_token](#output\_metering\_token) | Access token of the metering service account, or null when metering is off. | +| [platform\_identifier](#output\_platform\_identifier) | Platform identifier in the `.` form meshStack uses to address a platform, for example `ske-namespace.eu-de-central`. | +| [platform\_ref](#output\_platform\_ref) | Reference to the platform, for use in building block compositions that create tenants on it. | +| [replicator\_token](#output\_replicator\_token) | Access token of the replicator service account. meshStack already holds this token, so you only need the output to debug the cluster connection. | + diff --git a/modules/kubernetes/platform/buildingblock/logo.png b/modules/kubernetes/platform/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..d60632518f0d5128dc150eec655ba1aa740bfef3 GIT binary patch literal 10107 zcmdU#WltPj6NY!$#bt3T?k+73#ogVZNO3Rji@Up9OK~Wr6f5rT6xYSwowv`=c$3M= zWIjyhn%t9!008Hps;DU?2uHQonGW&R^#PIw_Qo^y_EWk_}Yux``5VYi{GQSju|J* zu*c@H+lQyupF=lVVF&F~cR}CIuJ2wAqYpjvPJN3`-EvR$qYkInA57zqgx&Yje_mQ9 zA8UjjSf?B>?L5l*?Zb|rAD>?Ho3F&Z_Kq)JHV&T1jkgs8_e=g<)%M@)o;)vZKNhxL zeeb+3Xt}aaKbc&8;IxPR7`V|6KioNbUf6odG1b@V_3v06}n%@c5SxG$BhTHV0fy{!$yn^Dv6U zK@JSZsrFGpF{az^+9jz-S#?;-(ekM3Mkro=4OOmY`FhtwL)tqzTSrbCva^A*>X64leBu_c(&_qirrarPFf zXlk=67qm^-!t!?)e5`nzY4xOI&iZVbOayKdDWj+RaCen;cKP6UhS}?@M?@T)S#Rxb zD(0aqws|O}fked~a&MB3U~crgJQl6Ce(vY_KsE&NHJTR5LtC3ix*#tt?A7pGii1NK zaNqpNpG10sJw9X_JsVjC82+|cyhxLkB^9rm8QgtPpk$5WoM&(aGX#r7;Og>c(eV2! zAQw|Ob$1tVx$9HiQ=(WZ=5bLn4oaQxjj_ujh)9Zc?YYvCz&AI8f7ojY2Nx(sSJ_K; z?PzJ-uPfr-@7u5$D$56v7gIdVgyDq4k&G(nD1?(2Lk<79Q4LBcua4=H^wRiSm^Vfb z{1MubTqk(-Rh-vz-WM z_(#5R6yNR%n_**T^U;M4jcL~=hAsM+eH!>F0yw8XR(nW;vG3RlA31mWvQLZ zRJHMo(4a(HS6IO=1L{A3`Gt{wK6pOK7@m~j9>_?~n3TCL*_QT=+zO9=gz=!9tDduwPRv?CTx>C_i0_eb2^K%y|reQS$Epqb}=#sQoL8ta#(a zn%Z&l!M)VS;Zruppr6VmCPW}6{d=_!g}#kl2~3wPo(;YU$hw-! zONt;2Fct_IA1q#>_kgR^wu2AxTuH^gtaFp78OCLJzmK*c@U1kcR4g9b!kqKk*OnH;WKF7;1=2<8qsa! z{F$2ubryv?THnADm>ruaGd>pd}W z959Dmhe+&2T8Ov+r+U(Ak<4t7)}^j~vMoFdM)3X@=FORP1Tnzj%c=wknJH%t(?W~$ z?CI++uK>(m0h4mgnrv|JDKsAv6{@JUt^jPr;?xH`2B-c5t)?V!Zy0rr@+dD6K0%2R zo~6ozx5*u^2w~fEby&xeVt=~>rjG2xgOt z>2!;dqR9Bmq6}dDWi~Tu=(-1U7rLK33`z|X2fldUk3yGsQe=S(dXmVxY;geeBu)bs zcr(%6`7CU`7YckzI~H1J>T$46S2{8>&j;KJ^x*MXHY0h{Vc7H5h*~K z{ql$41mk02{(xM6bHk6vP6uYBJZPyj@bRU2z`e0oNJH}mgM>|gr?Mrb!tqf`M;E7U=UeddkEP7=Q;L}Pvj^&

mnAg&;M&I;ueB^6N53!ydP#M4~_*{)$6<+%Mhd zGNgQ-C`KT10Fodj7QL+261g-`675>%WK`FN2%F_qs4jh}W}9W3Pd7?GiV|D8XIFl{ zi^P8U#nw(1G2Rfb@QxFQHtvM)wddX)uW+U`lzf;1KZ+Q*)cD2_Kx2x~00R2fNs4*S zp;9D{tSrloV^=@rg837Y{(<~S#qvL_G0(q-yhO3uG{oaLQ~CyZkt5Ah039Ww|_w9g(Y_sf!O!VR^CT zXW-V~Xl8IBm5{FZ?`MmSAqID!bVeU(ro&J(^n}G=T&EqzgYDrCyz5gFJwZGth8FYD zj>O24BG4-qIasZNNNOPjh-)~U2eJ8_k31#4+fQa9``J6uVohUWQkwGPi%u2pX+4r3 z>SJcjd~ZC8k;xfPhf)8t?kr?T1m{ckSgokHIF9SGy+<@Z6PwF8N;|#O;>&VioJt7C zt5GSF_dtbN07KXZN@_elWU!|-Jl;tVSei_LxVY7D>#nb8!S#KYW)X>D8~xqA=qLn+ z)mJ|e%CK~qhY8Z2T+-ckqrJotG((o<(p0vhul3l%GE&g6U~e;T5Z)_fx^H z4(XxqLPyJW;go3u^k#<)52B9P{yw*HEy7NIoCni;dgO#JQ< zbqd-+!P`%dY89o=dTm5Tb;H%wt1opL+~YOtrM7mq?TRP!v!fY#6*;GFEi-ZvZ32+N z5E%rZ+LRimn5JfD7_bkqJv;-0sRl?s`X~zjY4CWxVO37{^iT8Zf$uL7Mw-nv*NKO` zE5W^$6n=7vi_vVzoO7TRbMy^w&?OXh^6J)YuOn^?kJtHmSg7@I8F5%RA5mu?i-I1? zJ_fHkHus$LjfGh9rg;af!f44r!BtZP*EG$LIjf~{Hx7UUH+A8D1Z!?cEH{bRTXX`- zJKz(=ff?^uj5DO7?72bKBpjlRR+RZ`fmMS7YVZCE4o`lv8H%WA3jCo9-7?^odpl}3 ztt!Zz#0UK>16YE-j)nS9GyXiTP>z{K8?(^%iwsBZ5?ggDg+criBs_&^SYoNmHsO?9 zg0mwpN2!8v2nzPle%DmrCw7Gr>YHOLb-Kl^={uV-Y3)LOd3gb0aHyO0v5u-Y30a?Y zP&VVt8)K-xx2=s@T$?D?}!cc_{3FaT7GlN!UU}kw^2J^A)+cGui{11ab&$$ zMZaSVfuk7uXwE)<89R9HOH{oUljOUg3fy455@)&TiTXkniJR!cC9AxPiSuTj7 z1E3FT-lv$yyrBLSB6?-?*TqZB=tFFJ%&km3O1&O2X%P$wYLB=McoNbNGbgnF0s*spF@ z>6&_bamKhMWe8(NgCj7H*fmIibc^J`7d12mhszDcwL~c?N9ul#7E9h^c_F(1-SqXz zK#q&HU+&>6x?P|>9DZ33!C4OL?g%tG$s=&sER7#Yq5*#F(nroDL@W!%Eh1z$}C4@o0$ROmXA#J|GIU*%E`JET9bhmV4GC>EihO4i&baI z?xSw^yhCl?HykhrNx)LbfFhi2qeRD!8Nw@9ZtjJ8N{JuPZAH{tDDPN-wK(y!D_NKLmIU{;A`+|9CVnC+ZUMJ2Pv!CA5jMO@LG14UTto=2ze{1g(qEu7p5}(g-IwGc=-Om>Nk~n{Yev$k0!wnR=e>B z`W$>R_Pyv`n-NJwCGbJj@p-xontd->#Biz+#|SEs*@USo#%3Yi0*MaDVa#-~Lc2BJA0yaSTLo&^3e+gWQ!N23w&AvTcPW#I`^W5%95$l{xeoP3+&Ck@xz|cv7^v@I=?S9@aO-1={?~Z%7(r#{ z=Wy>I`m_0eMAOnf;rxy8P~G2J z!V$%kQ1i5Kk(t6w_m@iv!d{|ECbTb#3%x!G(3ytXp1!lC3xyAsq(}zS;w#bb#I|m1gbG;fLSlGIm`FZ7=e)xhi#Adv!TxmMH;Nz!Nq1I z{@2obN2vX^e+YX?yUlrNTfz}R{eix09;=YSc5o?@bx2goH&cvV2{c6{_Ol|mNjwq! zG;`K>GOih$_m>#;QWr#PoEJ!`1oLsc^W!{iXQ)NdP;9l*pimBSGZXP}gi%rA4Y*cR z(JIE^GM~d%rwbzF|8#&ms~tZ_Nfp2Txmy;ebgkeWqh~?t@gVLCA}IJp zh9PevQ?m0-1^h8>teKu7fVq2c0cZug5AziYFSVY6saO=1e;A8Nu;KA3S)lBo|s|Lk0pQc>Kfl_sSckgV^5fQ_eoJqQ>jxC=wA zHoLT3?s5l`s<>fyoqARk95Sw9ivZc#fb3UX?aL6TIF|k=Lb@e;HVHBk18l4x>IPO zW51cW!))K=(&h5O(4xKC?L4ZpSYIRZls=pRsev$WyI@@~PHOR)TSKWRO@P<_YCrp! zX@|_Zk0B$*2;S_|>2Q1x^4eM?>`x14kPc>w?Vnr}w}~@y!3JM;r6yg}n(G)ml1&kJ zpr4Vcvt~oe?CzF3)@*alp6FTS_-$w@&~t!3zV=_5Po{3V`({~NEnf&XcBVL!*e27Eyr$K-la^bgO&~wh zTBQ^nC-HhWAVX^Fu>?O~o_V#-y&`xZ@%roE|8CvqZ#U`?7sZep1dyGs_EGbJHpI@l z>V7DW#fl~@MUv(f4&*Kqne8hi6rpcFx;prmB}SZ6(btnyVj?};2;SL*y_i1?tYq>2 zSzBj`a;+pF>vs2{=QCZb{U#asV;f;KoH13~4O}i~DWk0u5M5FIVQ(IB3hnqUr}g%X zA(~}n2ObBhE=H8125$2kb1)km%KmA2c7FbWqSabT)=)?&7!E887j&*087+j56fRYQ z+Js61k2S;=lM6zkQs{K%dsbSWyE?21P*hpZr?~++LuU?J{EsRtd^?_2iDy%&HMN7S zKFoYEp%{T!r1sFT)-}fI92yWy-0+=l8{dvBvWGGe^XM!7+4Tq#F8R!^Rk9^HVEnj6 ziNze!7I@URr(FN3bY;ViYcg5E!j)Jf?+6$3=oxplit;Juex=5DU3e!%H+sB2m-kxM zoAYf-bRrIQRaM5{kXygrKst)7Vt+oa2C)Qe*T>p}gZ=z_qLG$&gxCu7c(j{KA7o2s zr95~f;?Hq2A(G z-eyUsEC-Qi2BDxi9B=5EE?zhyXb~{|D?$B{I%FDlSIA?AD30%hCwE3`O}vB z3qI2IZ+4@0lYLh2gz|fTVjQ}umefKjS5mG$eB$?SvW^aodplFF4^p)Os|y}L_wuhe zMjLYQcmyzCt@fgx-^k1{{gMWq#JACiL2X}ua`|7vxwAYN|18a`U{C2!-yyyUZ46NQ zJ^`_*0k-V6Adn&jJz}PAg^e!PSz_Zd;haXKKTX#v`@xKtbh`aIk`*g64w0?~aNnjf zvn`oe?$f=9Lb){x7d!xYGZcK4(Z?{?ueJ$av(W91LpKNA;o(}QOyI6hw`SK(wY9sK zY*df^FZZr&(_H61GClmm_TPS5mO22G7PN@0!t-K*JqMYAZmCn2UC%{ z>x~B2??F_Be4A1>>6*>lSo{y1Rykinu-DH9OJa8f-kp`iqfj3jWx#+}Pq1v8Xitp`YaT#E8IP4E^4@R^6zdo<+g zfkEOb$AT!OL!YJ1+8LJpq8{r7HZgd_6)xK_bsLS_q{^rn-@t3ZKPujNH{-R}ZQZ;i z7`hEAgdVS?Sg?uGvk*M>=!0zhohwYcVFKq* zD$_)rzQ#A;O}`kaK53KwPrN5d%RV$hO!;rAzuelhF&>G_D12}&_=a?C;%r~vQ0H7J z-fFYIv(oVddBb~AU86|YqBFuHvJ~PR5A#j=f78x9v_UWXE(}}*^h&%P`6MU9MtwGn z_ifq}Vl%;PtJp`1uMHy{flTA;I9J^O=QLsf4E`%dCVXfB^-v_>kXN`f?7^|**%b6x z#fAXx^h6I$llokSpx?+8Iq=r% zOz&X{)cZY_i9^~G&=tXbkK_Jz0uw`Ot3BZ3Q2#+Ys~!%%~P4e1@vuXTQ#$(qevs2DF)-LQp5r=;kMC0@gk z8`s)m$jYPWM$4S<>ElVykEYJ6Hi{Q)h{Kv&^;1R{LJ2JJGXcbYz%Y7!7O}GmBHGiN zi5?a(ZGd1?6KJv*-!AA`oBxhX*z@MJs2xP_L{$=uI+ZWTBh_7ZM~=hB7JyIZamOeE4Js$#1G3M9_TGXh0hTd?+B)JoVn%$^550*bhDXB;ku-}L2;gpF zUV)~VflloeDX+)(8V;U)QgvSmSNwA0el>bh8f2btNnkjPROZCrK6)y0FZvi5(mQpx z*1A^Jy~UH_I~S41qafqo3($VM|A+%f{L#t9fEKxZ9>^ZjdP=89&kk{&%y8Mz+ufT; zRWcK<>w@!|Y>CaJQX3Wyoe1RrP(IoKl&sR_A;qUCPFec}rKRXY2mGZ-mL74#)8(*h z*0OK=3a`+lh+E+H+)vVs+Pt@1bQIPL!i>3?#pBkgL=?0=c6>Te=b|=%!Xb!Tn?Njl z=0K8qm@8frB@9P6uw=5EJ_0ie(fp9OgdvdqCDyUBd4sBL{P8(ub%6y~B(zw~c-}Px z5?7zAeSBxtTOljHN=(`x^kzNxrXf1WQnv#lQvU8pz^TE9Et2LDM4yf!A>~EclL|yL zNfYgDBvbMaq;FxUo99iwb{~%NFxY3+K?8KMpy^Uzy1Xl9d=N@T{t*ycK4M6}QdapP zy~BsY7!3$G0#g+i5OQb$)2_sbyea7g(s=2=B27e0tfK@)I*cQo3aI&h*ubT)Y(OnM z6rXD`(6KD>P!{-l7<~CbbXok z?HNguC6Lzv5OP`DHX;7NC~G$Z6&1D4{>Qm8!nTY7Bpgew2GR>y-r8WO6MV8P0bWvK z{&;pi#N22se6A%^awzYV?QDo_$wssppz`XY!SwN4PHAIUP-+5wWl~;E=C}Lz0TzV- znZ}^tQlK!SewW0&f%~WM&fHl%q_9&(9T&YQC{hnm*Mc6PPl5Rm{uR$zMB6C@$Z_k5 zgIwge2tN0#IXe1jjg22mu6e*OUHwh=8+q^KtVI1~Zn z*n$Rk;W+|X_6le)wPIJFneB^#Kx9cgbKbWw9;+@^*ckJ`FdSafmp(#4C@x8+5UF6l z2|I)P#zBehPgWjJgP}@GB&; XO!Hj3fcW;`QWHQ lz.metadata.name } +} + +output "landing_zone_refs" { + description = "meshStack landing zone references keyed by environment, for use in building block compositions." + value = { for env, lz in meshstack_landingzone.this : env => lz.ref } +} + +output "replicator_token" { + description = "Access token of the replicator service account. meshStack already holds this token, so you only need the output to debug the cluster connection." + value = local.replicator_token + sensitive = true +} + +output "metering_token" { + description = "Access token of the metering service account, or null when metering is off." + value = local.metering_token + sensitive = true +} diff --git a/modules/kubernetes/platform/buildingblock/provider.tf b/modules/kubernetes/platform/buildingblock/provider.tf new file mode 100644 index 00000000..f3936b3c --- /dev/null +++ b/modules/kubernetes/platform/buildingblock/provider.tf @@ -0,0 +1,9 @@ +# Callers that drive this module from Terragrunt usually replace this file with a generated +# `provider.tf` of their own. In that case the three credential variables stay unset and the +# generated block carries the credentials instead. +provider "kubernetes" { + host = var.kube_host + cluster_ca_certificate = var.cluster_ca_certificate + client_certificate = var.client_certificate + client_key = var.client_key +} diff --git a/modules/kubernetes/platform/buildingblock/variables.tf b/modules/kubernetes/platform/buildingblock/variables.tf new file mode 100644 index 00000000..19cda78b --- /dev/null +++ b/modules/kubernetes/platform/buildingblock/variables.tf @@ -0,0 +1,269 @@ +variable "kube_host" { + type = string + nullable = false + description = "URL of the Kubernetes API server, for example `https://k8s.example.com:6443`. meshStack calls this URL to replicate tenants and to collect metering data." +} + +variable "cluster_ca_certificate" { + type = string + nullable = true + default = null + description = "PEM encoded CA certificate of the cluster. Leave unset when the caller supplies its own provider configuration." +} + +variable "client_certificate" { + type = string + nullable = true + default = null + sensitive = true + description = "PEM encoded client certificate the module authenticates with while it creates the in-cluster service accounts. Leave unset when the caller supplies its own provider configuration." +} + +variable "client_key" { + type = string + nullable = true + default = null + sensitive = true + description = "PEM encoded client key that belongs to `client_certificate`. Leave unset when the caller supplies its own provider configuration." +} + +variable "owning_workspace_identifier" { + type = string + nullable = false + description = "Identifier of the meshStack workspace that owns the platform and its landing zones." +} + +variable "location_identifier" { + type = string + nullable = false + description = "Identifier of the meshStack location the platform is registered in." +} + +variable "platform_name" { + type = string + nullable = false + default = "ske-namespace" + description = "meshStack platform identifier. The landing zones derive their names from it, for example `ske-namespace-dev`." +} + +variable "platform_display_name" { + type = string + nullable = false + default = "Kubernetes namespace on SKE" + description = "Name of the platform as users see it in meshPanel." +} + +variable "platform_description" { + type = string + nullable = false + default = "Provides a kubernetes namespace on STACKIT Kubernetes Engine (SKE)." + description = "Description of the platform as users see it in meshPanel." +} + +variable "documentation_url" { + type = string + nullable = false + default = "" + description = "Link to the platform documentation shown in meshPanel." +} + +variable "support_url" { + type = string + nullable = false + default = "" + description = "Link to the support channel shown in meshPanel." +} + +variable "namespace_name_pattern" { + type = string + nullable = false + default = "#{workspaceIdentifier}-#{projectIdentifier}" + description = "Pattern meshStack uses to name the namespace it creates for a tenant." +} + +variable "disable_ssl_validation" { + type = bool + nullable = false + default = true + description = "Skip SSL validation when meshStack calls the Kubernetes API server. SKE clusters serve a certificate that meshStack does not trust by default, which is why this is on." +} + +variable "service_account_namespace" { + type = string + nullable = false + default = "meshcloud" + description = "Namespace that holds the replicator and metering service accounts." +} + +variable "resource_name_suffix" { + type = string + nullable = false + default = "" + description = "Suffix appended to the in-cluster resource names. Set it when one cluster carries more than one meshStack platform registration, so the service accounts and cluster roles do not collide." +} + +variable "metering_enabled" { + type = bool + nullable = false + default = true + description = "Create the metering service account and register metering on the platform. Turn this off when meshStack should not collect usage data from the cluster." +} + +variable "metering_processing" { + type = object({ + compact_timelines_after_days = optional(number, 30) + delete_raw_data_after_days = optional(number, 65) + }) + nullable = false + default = {} + description = "How long meshMetering keeps timelines and raw data. Only used when `metering_enabled` is true." +} + +variable "replicator_additional_rules" { + type = list(object({ + api_groups = list(string) + resources = list(string) + verbs = list(string) + resource_names = optional(list(string)) + non_resource_urls = optional(list(string)) + })) + nullable = false + default = [] + description = "Extra RBAC rules added to the replicator cluster role." +} + +variable "metering_additional_rules" { + type = list(object({ + api_groups = list(string) + resources = list(string) + verbs = list(string) + resource_names = optional(list(string)) + non_resource_urls = optional(list(string)) + })) + nullable = false + default = [] + description = "Extra RBAC rules added to the metering cluster role." +} + +# Cluster sizing behind the defaults: 2 vCPU + 8 Gi RAM per node, 1 node default and 3 nodes max. +# Expected density is 20-30 namespaces across the cluster. CPU is given in millicores (m) and +# memory in mebibytes (Mi) so every value stays a whole integer. +variable "quota_definitions" { + type = list(object({ + quota_key = string + label = string + description = string + unit = string + min_value = number + max_value = number + auto_approval_threshold = number + })) + nullable = false + description = "Quota keys a tenant can request on this platform, with the upper bound and the threshold below which meshStack approves a request automatically." + + default = [ + { + quota_key = "limits.cpu" + label = "CPU limit" + description = "The sum of CPU limits across all pods in a non-terminal state cannot exceed this value." + unit = "m" + min_value = 0 + max_value = 1000 # 1 vCPU per namespace + auto_approval_threshold = 1000 + }, + { + quota_key = "requests.cpu" + label = "CPU requests" + description = "The sum of CPU requests across all pods in a non-terminal state cannot exceed this value." + unit = "m" + min_value = 0 + max_value = 1000 + auto_approval_threshold = 500 + }, + { + quota_key = "limits.memory" + label = "Memory limit" + description = "The sum of memory limits across all pods in a non-terminal state cannot exceed this value." + unit = "Mi" + min_value = 0 + max_value = 1024 # 1 Gi per namespace + auto_approval_threshold = 1024 + }, + { + quota_key = "requests.memory" + label = "Memory requests" + description = "The sum of memory requests across all pods in a non-terminal state cannot exceed this value." + unit = "Mi" + min_value = 0 + max_value = 1024 + auto_approval_threshold = 512 + }, + { + quota_key = "requests.storage" + label = "Total Storage Requests" + description = "Across all persistent volume claims, the sum of storage requests cannot exceed this value." + unit = "Gi" + min_value = 0 + max_value = 5 + auto_approval_threshold = 2 + }, + { + quota_key = "persistentvolumeclaims" + label = "Persistent Volume Claims" + description = "The total number of PersistentVolumeClaims that can exist in the namespace." + unit = "" + min_value = 0 + max_value = 4 + auto_approval_threshold = 2 + }, + ] +} + +variable "landing_zones" { + type = map(object({ + display_name = string + description = string + info_link = optional(string, "") + tags = optional(map(list(string)), {}) + quotas = optional(list(object({ key = string, value = number })), []) + })) + nullable = false + description = "Landing zones to create for the platform, keyed by environment. The key becomes the suffix of the landing zone identifier, for example `dev` gives `ske-namespace-dev`." + + default = { + dev = { + display_name = "SKE Kubernetes Namespace – Development" + description = "Landing zone for development workloads." + tags = { + "LandingZoneFamily" = ["cloud-native"] + "environment" = ["dev"] + "confidentiality" = ["internal"] + } + quotas = [ + { key = "limits.cpu", value = 500 }, + { key = "requests.cpu", value = 250 }, + { key = "limits.memory", value = 512 }, + { key = "requests.memory", value = 256 }, + { key = "requests.storage", value = 1 }, + { key = "persistentvolumeclaims", value = 2 }, + ] + } + prod = { + display_name = "SKE Kubernetes Namespace – Production" + description = "Landing zone for production workloads." + tags = { + "LandingZoneFamily" = ["cloud-native"] + "environment" = ["prod"] + "confidentiality" = ["public"] + } + quotas = [ + { key = "limits.cpu", value = 1000 }, + { key = "requests.cpu", value = 500 }, + { key = "limits.memory", value = 1024 }, + { key = "requests.memory", value = 512 }, + { key = "requests.storage", value = 2 }, + { key = "persistentvolumeclaims", value = 4 }, + ] + } + } +} diff --git a/modules/kubernetes/platform/buildingblock/versions.tf b/modules/kubernetes/platform/buildingblock/versions.tf new file mode 100644 index 00000000..a2689284 --- /dev/null +++ b/modules/kubernetes/platform/buildingblock/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.38.0" + } + + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.20.0" + } + } +} From b4cd016da7763dc2991ae39fd60e6bc906acf620 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:13:07 +0200 Subject: [PATCH 18/60] feat(stackit/model-serving): finish the module as a publishable building block Adds the meshstack_integration.tf that was missing, so the module can be imported into a meshStack instance. The building block definition is TENANT_LEVEL and mints one token per tenant rather than sharing a single token across the platform. Its readme is inline, as the convention requires for modules that carry an integration file. Adds buildingblock/logo.png as a copy of modules/stackit/logo.png, which is what stackit/project and stackit/ske already do for building blocks without a service-specific logo. Exposes an api_base output that carries the '/v1' suffix, because a caller that drops the suffix gets a "Not Found" error from the OpenAI-compatible endpoint. Together with the token, that output is everything LiteLLM needs to register STACKIT as a model backend: LiteLLM sends api_key upstream as a bearer token in the Authorization header. A summary output renders both for the application team. Fixes the ttl_duration default. STACKIT validates it with Go's duration parser, which knows no day unit, so the previous '90d' was rejected at apply time; 90 days is now written as '2160h'. Lowers required_version to >= 1.11.0 to match the terraform_version the building block definition runs and every other STACKIT building block. The module uses no feature newer than that. Declares requiresBackplane: false: the token is created in a STACKIT project that already exists and the service account arrives as an input, so the module provisions nothing cloud-side of its own. This pull request adds no e2e tests, so the two e2e scorecard checks stay open by decision. Scorecard: 40% -> 90%. Co-Authored-By: Claude Opus 5 (1M context) --- .../model-serving/buildingblock/README.md | 22 +- .../buildingblock/SUMMARY.md.tftpl | 23 ++ .../model-serving/buildingblock/logo.png | Bin 0 -> 1878 bytes .../model-serving/buildingblock/outputs.tf | 19 ++ .../model-serving/buildingblock/variables.tf | 8 +- .../model-serving/buildingblock/versions.tf | 2 +- .../model-serving/meshstack_integration.tf | 251 ++++++++++++++++++ 7 files changed, 317 insertions(+), 8 deletions(-) create mode 100644 modules/stackit/model-serving/buildingblock/SUMMARY.md.tftpl create mode 100644 modules/stackit/model-serving/buildingblock/logo.png create mode 100644 modules/stackit/model-serving/meshstack_integration.tf diff --git a/modules/stackit/model-serving/buildingblock/README.md b/modules/stackit/model-serving/buildingblock/README.md index d4678a55..828eb257 100644 --- a/modules/stackit/model-serving/buildingblock/README.md +++ b/modules/stackit/model-serving/buildingblock/README.md @@ -3,20 +3,32 @@ name: STACKIT AI Model Serving Access supportedPlatforms: - stackit description: Issues a scoped STACKIT AI Model Serving API token so a tenant can call the sovereign LLM API. +# The module creates the token in a STACKIT project that already exists and receives the service +# account it authenticates with as an input, so there is no cloud-side setup to perform ahead of time. +requiresBackplane: false --- # STACKIT AI Model Serving Access Building Block - - This building block issues a STACKIT AI Model Serving API token scoped to a tenant's STACKIT project. +Each tenant gets its own token rather than sharing one token across the platform, so the platform +team can revoke a single tenant's access and read its usage separately. + +The token authenticates calls against STACKIT's OpenAI-compatible inference endpoint. The `api_base` +output carries the full base URL including the `/v1` suffix, because a caller that drops the suffix +gets a "Not Found" error from the endpoint. Callers send the token as a bearer token in the +`Authorization` header, which is what LiteLLM's `api_key` field sends upstream β€” so registering this +endpoint as a LiteLLM backend needs nothing beyond the `api_base` and `token` outputs. + +STACKIT parses `ttl_duration` with Go's duration parser, which knows no day unit. Write 90 days as +`2160h`; a value such as `90d` is rejected. ## Requirements | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [terraform](#requirement\_terraform) | >= 1.11.0 | | [stackit](#requirement\_stackit) | >= 0.88.0 | ## Modules @@ -38,12 +50,14 @@ No modules. | [service\_account\_email](#input\_service\_account\_email) | Email of the STACKIT service account used to issue the Model Serving token. | `string` | n/a | yes | | [token\_description](#input\_token\_description) | Description shown on the Model Serving token. | `string` | `"Managed by meshStack."` | no | | [token\_name](#input\_token\_name) | Display name of the Model Serving token. | `string` | n/a | yes | -| [ttl\_duration](#input\_ttl\_duration) | Lifetime of the Model Serving token, e.g. '90d'. | `string` | `"90d"` | no | +| [ttl\_duration](#input\_ttl\_duration) | Lifetime of the Model Serving token as a Go duration, e.g. '2160h' for 90 days. Valid units are 'ns', 'us', 'ms', 's', 'm' and 'h'. | `string` | `"2160h"` | no | ## Outputs | Name | Description | |------|-------------| +| [api\_base](#output\_api\_base) | OpenAI-compatible base URL of the STACKIT inference endpoint, including the '/v1' suffix. | +| [summary](#output\_summary) | Summary with the endpoint URL and the token. | | [token](#output\_token) | The STACKIT Model Serving API token. Shown only on creation. | | [token\_id](#output\_token\_id) | ID of the Model Serving token. | | [valid\_until](#output\_valid\_until) | Expiry timestamp of the Model Serving token. | diff --git a/modules/stackit/model-serving/buildingblock/SUMMARY.md.tftpl b/modules/stackit/model-serving/buildingblock/SUMMARY.md.tftpl new file mode 100644 index 00000000..68e5d037 --- /dev/null +++ b/modules/stackit/model-serving/buildingblock/SUMMARY.md.tftpl @@ -0,0 +1,23 @@ +# Model Serving Token: **${token_name}** + +## Details + +| Property | Value | +|----------|-------| +| **API Base URL** | `${api_base}` | +| **API Token** | `${token}` | +| **Token ID** | `${token_id}` | +| **Valid Until** | ${valid_until} | + +## Calling the endpoint + +The endpoint is OpenAI-compatible and the base URL ends in `/v1`. Send the token as a bearer token +in the `Authorization` header: + +```sh +curl "${api_base}/models" \ + -H "Authorization: Bearer ${token}" +``` + +Point an OpenAI client library at the base URL and pass the token as the API key. The token is shown +here only once, so store it in your own secret store. diff --git a/modules/stackit/model-serving/buildingblock/logo.png b/modules/stackit/model-serving/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..225391782a085772b79e9ce3d491e62e933a940b GIT binary patch literal 1878 zcmZ8i2~?6<6#mQHCCjnK(l(cza%3*YfuRsEa0DL$4O$-xtp}b4MMWVo zfeMjAp^y-y03^Mv1$zUw z4poaVh9R*ztJC4?ulWj;;bZ0`Y)R7wF>gasHDB$$xQupA>x&glpWun7U!X z6yQ-nUiT1VI|15|w@CB~(tsaHH7}U30rKe!DYTvXWx~+sH{JTfGnX5D{Cm<)ZwzVm z26H|C-IKMKgpD=n^)GKu?72mac~WN-#!cL^CfN1oUA{|~ESk$PnQdWa!`#fD<*_~1 z;aOXD_4(g#yq0BUJiPGWeNNtJ$?>Kl(bf4s1{zPR>_|N_eSv2ZAv!MJ*)eL0U2-HZ z!g7_w>Li>k+$@!KgpE?hYsxj z<92SQebBlJK|$~0{N5)kPC(G4$#|bN8+WwzswtEWYb^Dj&vlsKow3)Xk$S+OZX$Jx zsZoKzX!<0dT5q@EXZ-%^k!So)>1bTu@R?<7QDe`XZExtwk9^tfx!YIzY!4ZFbstJw zImi!q?+fSOXT={=QI0D-HN1O^EwtwD*ac^5^IR$%OX5a8aJ9}BlAB)e96#2GbycBh zK+h@er#Q#-Y9Dx{q^i*W_3%9Vp-%!ePFTy>GEgYN_sBFu0a?8`5ksGGm$Vx`=%v{YowgOH%u7%C?e2AtBelE91ma+(vh#i~)A{CG zvN*-n7%P#g?+9ygkDrHRIEdKyyZ1Y*Qi|2H>sfADs*X%WB&J+zYoU#H$70Xc!UK&F z2J$50F;zliYql+K$wpXOv_tr`eqr5`DsFJ3Wy8RqXD;s)QtDls@0@M8V^tRaywr`n zO6KG^0VliqxYaQ>@5QOX>1A%6rK{2OHvKZ`FUs=hql1h?uE{o51{E5(^*oW9>bl~G)8J69EY7}oU3@HN8@=x^hCf3R zYPZM?mRCoYGUL6o<$4Hv_9jM)2~QkTQw;Nz9P6PZ-ZHl3A`C3gEoR31Quz92G8I=9 zr@XCbc`IN7pu`$swO;rQl&z0CO58kO5{5$>+RK>new09*Y0y3u*O8-C+1|CZ^$)3Z zdHOQKRGszL@*cu+ufB$mh{`?&VuEmqva1%ni&s}yaSI{<&rF@ zpIo7N+tl5oBJ-zpBz?8oD*xc;%@IF)aP8{_muwlkU&ax$DqSc(?0z{%Y*tqumQV?Q zF2GXYH0i>ad*@v!f!-^k4iHmgX$A$ABX?wCC+B5A&2q zf~v9-C8muV?wkZNhQ{r7U+b*Dgy7xHHAi#bMAaFH5#Pl1ls`I1qPza&yut}6k*-ZB zWtDvvHK~iJb`_=SB4q*zW{YW4x61YRYodHB&JjStC3PR5-ub`Y^Tq0-j{$_MVw#8m zBR&l7j}mptdjT|#V9^uR5o#oQ@f=t~|Nuh`oom fvBSwdOZPyP@64?#ZB$msKSI1O!RLl|#NK}ZQo#tq literal 0 HcmV?d00001 diff --git a/modules/stackit/model-serving/buildingblock/outputs.tf b/modules/stackit/model-serving/buildingblock/outputs.tf index 7fd10643..1b4100a3 100644 --- a/modules/stackit/model-serving/buildingblock/outputs.tf +++ b/modules/stackit/model-serving/buildingblock/outputs.tf @@ -13,3 +13,22 @@ output "valid_until" { value = stackit_modelserving_token.this.valid_until description = "Expiry timestamp of the Model Serving token." } + +output "api_base" { + value = "https://api.openai-compat.model-serving.${stackit_modelserving_token.this.region}.onstackit.cloud/v1" + # The '/v1' suffix belongs to the base URL. A client that drops it gets a 'Not Found' error + # from the endpoint, so this output carries the suffix instead of leaving it to the caller. + description = "OpenAI-compatible base URL of the STACKIT inference endpoint, including the '/v1' suffix." +} + +output "summary" { + description = "Summary with the endpoint URL and the token." + sensitive = true + value = templatefile("${path.module}/SUMMARY.md.tftpl", { + token_name = stackit_modelserving_token.this.name + token_id = stackit_modelserving_token.this.token_id + token = stackit_modelserving_token.this.token + valid_until = stackit_modelserving_token.this.valid_until + api_base = "https://api.openai-compat.model-serving.${stackit_modelserving_token.this.region}.onstackit.cloud/v1" + }) +} diff --git a/modules/stackit/model-serving/buildingblock/variables.tf b/modules/stackit/model-serving/buildingblock/variables.tf index 62fd2b4e..2d5e9557 100644 --- a/modules/stackit/model-serving/buildingblock/variables.tf +++ b/modules/stackit/model-serving/buildingblock/variables.tf @@ -20,9 +20,11 @@ variable "token_description" { } variable "ttl_duration" { - type = string - default = "90d" - description = "Lifetime of the Model Serving token, e.g. '90d'." + type = string + default = "2160h" + # STACKIT parses this with Go's duration parser, which knows no day unit. '90d' is rejected, + # so 90 days has to be written as '2160h'. + description = "Lifetime of the Model Serving token as a Go duration, e.g. '2160h' for 90 days. Valid units are 'ns', 'us', 'ms', 's', 'm' and 'h'." } variable "region" { diff --git a/modules/stackit/model-serving/buildingblock/versions.tf b/modules/stackit/model-serving/buildingblock/versions.tf index 249d5995..706aef93 100644 --- a/modules/stackit/model-serving/buildingblock/versions.tf +++ b/modules/stackit/model-serving/buildingblock/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.12.0" + required_version = ">= 1.11.0" required_providers { stackit = { diff --git a/modules/stackit/model-serving/meshstack_integration.tf b/modules/stackit/model-serving/meshstack_integration.tf new file mode 100644 index 00000000..93f933c0 --- /dev/null +++ b/modules/stackit/model-serving/meshstack_integration.tf @@ -0,0 +1,251 @@ +variable "stackit_service_account_email" { + type = string + description = "Email of the STACKIT service account the building block authenticates with via workload identity federation. The account needs permission to create AI Model Serving tokens in the tenant projects this definition targets." +} + +variable "stackit_region" { + type = string + default = "eu01" + description = "STACKIT region the Model Serving token is issued in. The region is part of the inference endpoint URL." +} + +variable "stackit_token_ttl_duration" { + type = string + default = "2160h" + # STACKIT parses this with Go's duration parser, which knows no day unit. '90d' is rejected, + # so 90 days has to be written as '2160h'. + description = "Lifetime of every token this definition issues, as a Go duration. '2160h' is 90 days. Valid units are 'ns', 'us', 'ms', 's', 'm' and 'h'." +} + +variable "meshstack" { + type = object({ + owning_workspace_identifier = string + tags = optional(map(list(string)), {}) + }) + description = "Shared meshStack context. Tags are optional and propagated to building block definition metadata." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { + git_ref = "main" + bbd_draft = true + } + description = <<-EOT + `git_ref`: Hub release reference. Set to a tag (e.g. 'v1.2.3') or branch or commit sha of meshcloud/meshstack-hub repo. + `bbd_draft`: If true, allows changing the building block definition for upgrading dependent building blocks. + EOT +} + +output "building_block_definition" { + description = "BBD is consumed in building block compositions, for example by the ai-platform reference architecture." + value = { + uuid = meshstack_building_block_definition.this.metadata.uuid + version_ref = var.hub.bbd_draft ? meshstack_building_block_definition.this.version_latest : meshstack_building_block_definition.this.version_latest_release + } +} + +resource "meshstack_building_block_definition" "this" { + metadata = { + owned_by_workspace = var.meshstack.owning_workspace_identifier + tags = var.meshstack.tags + } + + spec = { + display_name = "STACKIT AI Model Serving Access" + symbol = "https://raw.githubusercontent.com/meshcloud/meshstack-hub/${var.hub.git_ref}/modules/stackit/model-serving/buildingblock/logo.png" + description = "Issues a STACKIT AI Model Serving token so a tenant can call STACKIT's OpenAI-compatible inference endpoint." + support_url = "https://portal.stackit.cloud" + target_type = "TENANT_LEVEL" + run_transparency = true + supported_platforms = [{ name = "STACKIT" }] + + readme = chomp(<<-EOT + This building block issues a STACKIT AI Model Serving token in your own STACKIT project, so + your application can call STACKIT's OpenAI-compatible inference endpoint. Every project gets + its own token, so usage and revocation stay per project. + + ## 🎯 When to use it + + Use this building block when you: + - Want to call a large language model hosted on STACKIT from your application. + - Need an endpoint that works with the usual OpenAI client libraries and with gateways such as LiteLLM. + - Want a model credential that belongs to your project alone, instead of one token shared across the whole platform. + + ## πŸ’‘ Usage examples + + **Example 1: A chat feature in an application** + An application team orders this building block in their project and reads the API base URL + and the token from the outputs. The team stores both in a Kubernetes secret and points the + OpenAI client library of the application at them. + + **Example 2: A model backend in LiteLLM** + A team registers STACKIT AI Model Serving as a backend in LiteLLM. The `api_base` output goes + into the `api_base` field of the model entry and the token goes into the `api_key` field, + which LiteLLM sends upstream as a bearer token in the `Authorization` header. + + ## πŸ”‘ Calling the endpoint + + The `api_base` output already ends in `/v1`, and that suffix belongs to the base URL. A caller + that drops it gets a "Not Found" error. Send the token as a bearer token: + + ```sh + curl "$API_BASE/models" \ + -H "Authorization: Bearer $API_TOKEN" + ``` + + The token is shown once, when the building block runs. Copy it into your own secret store. + When it expires, order the building block again to get a fresh one. + + ## πŸ“Š Shared Responsibility + + | Responsibility | Platform Team | Application Team | + |---|:---:|:---:| + | Enable STACKIT AI Model Serving and provide the service account that issues tokens | βœ… | ❌ | + | Set the token lifetime that applies to every project | βœ… | ❌ | + | Store the token in the application's own secret store | ❌ | βœ… | + | Choose the model and carry the cost of the calls | ❌ | βœ… | + | Order the building block again before the token expires | ❌ | βœ… | + EOT + ) + } + + version_spec = { + draft = var.hub.bbd_draft + deletion_mode = "DELETE" + + implementation = { + terraform = { + terraform_version = "1.11.5" + repository_url = "https://github.com/meshcloud/meshstack-hub.git" + repository_path = "modules/stackit/model-serving/buildingblock" + ref_name = var.hub.git_ref + async = false + use_mesh_http_backend_fallback = true + } + } + + inputs = { + project_id = { + display_name = "STACKIT Project ID" + description = "STACKIT project ID of the tenant the token is issued in." + type = "STRING" + assignment_type = "PLATFORM_TENANT_ID" + } + + service_account_email = { + display_name = "Service Account Email" + description = "Email of the STACKIT service account for WIF-based authentication." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_service_account_email) + } + + STACKIT_USE_OIDC = { + display_name = "STACKIT Use OIDC" + description = "Enables OIDC-based WIF for the STACKIT provider." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("1") + } + + STACKIT_FEDERATED_TOKEN_FILE = { + display_name = "STACKIT Federated Token File" + description = "Path to the WIF token file injected by meshStack." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("/var/run/secrets/workload-identity/azure/token") + } + + region = { + display_name = "STACKIT Region" + description = "Region the Model Serving token is issued in." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_region) + } + + ttl_duration = { + display_name = "Token Lifetime" + description = "Lifetime of the token as a Go duration, for example '2160h' for 90 days." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_token_ttl_duration) + } + + token_name = { + display_name = "Token Name" + description = "Name of the token, shown next to it in the STACKIT portal." + type = "STRING" + assignment_type = "USER_INPUT" + value_validation_regex = "^.{1,200}$" + validation_regex_error_message = "The token name must be between 1 and 200 characters long." + } + + # STACKIT rejects an empty description, so the regex enforces at least one character. + token_description = { + display_name = "Token Description" + description = "Description of the token, shown next to it in the STACKIT portal." + type = "STRING" + assignment_type = "USER_INPUT" + updateable_by_consumer = true + default_value = jsonencode("Managed by meshStack.") + value_validation_regex = "^.{1,2000}$" + validation_regex_error_message = "The token description must be between 1 and 2000 characters long." + } + } + + outputs = { + api_base = { + display_name = "API Base URL" + description = "OpenAI-compatible base URL of the inference endpoint, including the '/v1' suffix." + type = "STRING" + assignment_type = "NONE" + } + + token = { + display_name = "API Token" + description = "The token the application sends as a bearer token in the 'Authorization' header." + type = "STRING" + assignment_type = "NONE" + } + + token_id = { + display_name = "Token ID" + description = "ID of the Model Serving token in STACKIT." + type = "STRING" + assignment_type = "NONE" + } + + valid_until = { + display_name = "Valid Until" + description = "Timestamp at which the token expires." + type = "STRING" + assignment_type = "NONE" + } + + summary = { + display_name = "Summary" + type = "STRING" + assignment_type = "SUMMARY" + } + } + } +} + +terraform { + required_version = ">= 1.12.0" + + required_providers { + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.21.0" + } + } +} From 43a7c6d7976ff8773c4b35bdda7e10bf66a7fbe5 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:22:55 +0200 Subject: [PATCH 19/60] feat(ref-arch): make stackit-kubernetes orderable as one cluster building block The architecture no longer expects an SKE cluster to exist. It now carries a meshstack_integration.tf and a buildingblock/ that composes three hub modules into one TENANT_LEVEL building block ordered against the STACKIT Project platform, so the cluster lands in the meshTenant's own STACKIT project: - modules/stackit/ske creates the cluster and its kubeconfig, - modules/kubernetes/platform registers it as a meshStack platform of type kubernetes with its namespace landing zones, - modules/kubernetes/ingress installs cert-manager, HAProxy and the Let's Encrypt ClusterIssuer. All three are sourced by git URL and pinned with ?ref=${var.hub.git_ref}. The tenant-facing input is one STRING, `expose`, taking public, internal or none. Everything else is a landing-zone concern and arrives as a STATIC input or stays at the module default. dns.tf holds the whole delegated-subzone design in one place, including the unverified assumption that NS delegation bypasses the one-label rule under stackit.run, and the modules/stackit/dns module the zone and the NS record still need. No cloud provider resource is declared here. Co-Authored-By: Claude Opus 5 (1M context) --- .../stackit-kubernetes/README.md | 106 ++++- .../buildingblock/README.md | 36 ++ .../stackit-kubernetes/buildingblock/dns.tf | 66 +++ .../stackit-kubernetes/buildingblock/logo.png | Bin 0 -> 22760 bytes .../stackit-kubernetes/buildingblock/main.tf | 54 +++ .../buildingblock/outputs.tf | 45 ++ .../buildingblock/provider.tf | 18 + .../buildingblock/variables.tf | 112 +++++ .../buildingblock/versions.tf | 18 + .../meshstack_integration.tf | 403 ++++++++++++++++++ 10 files changed, 835 insertions(+), 23 deletions(-) create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/README.md create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/dns.tf create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/logo.png create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/main.tf create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/outputs.tf create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/provider.tf create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/variables.tf create mode 100644 reference-architectures/stackit-kubernetes/buildingblock/versions.tf create mode 100644 reference-architectures/stackit-kubernetes/meshstack_integration.tf diff --git a/reference-architectures/stackit-kubernetes/README.md b/reference-architectures/stackit-kubernetes/README.md index db3bbd8a..6831ff34 100644 --- a/reference-architectures/stackit-kubernetes/README.md +++ b/reference-architectures/stackit-kubernetes/README.md @@ -1,13 +1,20 @@ --- name: STACKIT Kubernetes Platform description: > - A sovereign-cloud Kubernetes platform on STACKIT that provides application teams - with self-service SKE namespaces, Forgejo Git repositories, CI/CD via Forgejo Actions, - and a container registry backed by Harbor β€” all composed into a single starterkit - building block. + A sovereign-cloud Kubernetes platform on STACKIT. One order creates an SKE cluster, + gives it an HTTPS ingress with Let's Encrypt certificates and registers it in meshStack + as a Kubernetes platform with namespace landing zones. A second order gives application + teams self-service namespaces, Forgejo Git repositories, CI/CD via Forgejo Actions and a + container registry backed by Harbor. cloudProviders: - stackit buildingBlocks: + - path: stackit/ske + role: Creates the SKE cluster and the kubeconfig the rest of the architecture authenticates with. + - path: kubernetes/platform + role: Registers the cluster in meshStack as a platform of type kubernetes and creates its namespace landing zones. + - path: kubernetes/ingress + role: Installs cert-manager, the HAProxy ingress controller and a Let's Encrypt ClusterIssuer, and issues the wildcard certificate for the cluster's DNS subzone. - path: ske/ske-starterkit role: Orchestrates the full developer onboarding by composing dev/prod projects, SKE tenants, Git repos, and connectors into one self-service offering. - path: stackit/git-repository @@ -21,12 +28,14 @@ buildingBlocks: ## Overview The **STACKIT Kubernetes Platform** reference architecture delivers a complete, -sovereign-cloud Kubernetes experience on [STACKIT](https://www.stackit.de/). It combines -three Hub building blocks into a cohesive platform that gives application teams self-service -access to Kubernetes namespaces with integrated Git repositories and CI/CD pipelines β€” -all running on European infrastructure with full data sovereignty. Each team receives -a ready-to-use ai-summarizer demo application with provisioned access to STACKIT Model Serving, -a sovereign LLM API, ensuring even AI capabilities remain under full data control. +sovereign-cloud Kubernetes experience on [STACKIT](https://www.stackit.de/). The architecture +provisions the cluster itself: one order creates an SKE cluster, installs an ingress controller +with Let's Encrypt certificates on it and registers it in meshStack as a Kubernetes platform whose +landing zones hand out namespaces. On top of that platform it gives application teams self-service +namespaces with integrated Git repositories and CI/CD pipelines β€” all running on European +infrastructure with full data sovereignty. Each team receives a ready-to-use ai-summarizer demo +application with provisioned access to STACKIT Model Serving, a sovereign LLM API, ensuring even +AI capabilities remain under full data control. **Target audience:** @@ -40,13 +49,51 @@ a sovereign LLM API, ensuring even AI capabilities remain under full data contro ## How It Works -### 1. STACKIT Kubernetes Engine (SKE) - -SKE is a managed Kubernetes service provided by STACKIT. The platform team provisions -and maintains the cluster(s); application teams consume namespaces via meshStack tenants. -SKE handles control-plane management, upgrades, and scaling automatically. - -### 2. Developer Starterkit β€” `ske/ske-starterkit` +### 1. The Cluster β€” `STACKIT Kubernetes Cluster` building block + +SKE is a managed Kubernetes service provided by STACKIT, and SKE handles control-plane management, +upgrades and scaling automatically. The architecture creates the cluster rather than expecting one +to exist. Its `TENANT_LEVEL` building block is ordered against the STACKIT Project platform, so the +cluster lands in the meshTenant's own STACKIT project, and it composes three Hub modules in a +single Terraform run: + +1. **`stackit/ske`** creates the cluster and its kubeconfig. +2. **`kubernetes/platform`** registers the cluster in meshStack as a platform of type `kubernetes` + and creates the dev and prod namespace landing zones. Application teams consume namespaces on + it through meshStack tenants. +3. **`kubernetes/ingress`** installs cert-manager, the HAProxy ingress controller and a Let's + Encrypt ClusterIssuer. + +The team ordering a cluster decides two things: its name and how the ingress is reachable. The +`expose` input takes `public`, `internal` β€” which keeps the load balancer inside the STACKIT +network β€” or `none`, which installs no ingress controller at all. Everything else is a +landing-zone concern that the platform team sets once. + +### 2. Hostnames and Certificates + +Each cluster receives a **delegated DNS subzone** named after it, for example +`cluster1.likvid.stackit.run`, in the tenant's own STACKIT project. The landing zone owns the +parent zone and delegates the subzone with an NS record. The SKE managed ExternalDNS extension +writes the records from the control plane, and cert-manager holds a **single wildcard certificate** +for the whole subzone, which HAProxy serves for every application hostname. + +The result is a landing zone that promises more than a namespace: an application team that adds an +Ingress with a hostname under the subzone gets a working HTTPS URL, without requesting a +certificate and without creating a DNS record. + +> **Known risk β€” the delegated subzone is not verified yet.** STACKIT documents free `stackit.run` +> subdomains as one label deep, and it separately documents creating a subzone in a different +> project through NS delegation. Whether delegation bypasses the one-label rule *under +> `stackit.run`* is **unverified** and is being tested separately. If it does not, the fallback is +> a platform-owned zone: the parent zone stays in the platform team's project and each cluster gets +> records in it rather than a subzone of its own. The DNS handling is isolated in +> [`buildingblock/dns.tf`](buildingblock/dns.tf) so the fallback changes that one file. +> +> Creating the subzone and the NS record also needs a `modules/stackit/dns` module, which the Hub +> does not have yet. Until it exists, a platform team that wants the wildcard certificate creates +> the zone and the DNS-01 service account key by hand. + +### 3. Developer Starterkit β€” `ske/ske-starterkit` The starterkit is the **single entry point** for application teams. When a developer orders the starterkit from the meshStack self-service catalog, the following resources @@ -72,7 +119,7 @@ are created automatically: 6. **Project Admin binding** β€” the requesting developer is granted Project Admin on both projects. -### 3. Forgejo Git Repository β€” `stackit/git-repository` +### 4. Forgejo Git Repository β€” `stackit/git-repository` Each application team's repository includes: @@ -88,7 +135,7 @@ Each application team's repository includes: - **Template repository** β€” optionally cloned from a template URL, pre-configured with an ai-summarizer sample application that uses the STACKIT Model Serving API. -### 4. CI/CD Pipeline β€” `ske/forgejo-connector` +### 5. CI/CD Pipeline β€” `ske/forgejo-connector` The connector building block creates per-stage resources: @@ -112,21 +159,34 @@ The connector building block creates per-stage resources: |----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------| | meshStack instance | With Terraform/OpenTofu IaC runtime configured. | | STACKIT account | With access to SKE, STACKIT Git, and the global STACKIT Harbor registry. | -| SKE cluster | A running STACKIT Kubernetes Engine cluster with kubeconfig. | +| STACKIT Project platform | Registered in meshStack, for example through the [`stackit-landingzone`](../stackit-landingzone) reference architecture. Clusters are ordered on its tenants. | +| STACKIT identity | A service account with `ske.admin` on the organization and workload identity federation configured for the cluster building block definition. | | Forgejo organization | On STACKIT Git, with an API token for the Terraform provider. | | Harbor credentials | Robot account credentials (username and secret) for push/pull access to the STACKIT global Harbor registry; shared across all STACKIT customers. | | Model Serving API | STACKIT Model Serving endpoint and API key for the platform team to provide to the connector. | -| DNS zone | A DNS zone provided by STACKIT for application ingress hostnames (e.g. `apps.example.com`). | +| DNS parent zone | A STACKIT DNS zone the landing zone owns, for example `likvid.stackit.run`. Each cluster gets a delegated subzone under it. | + +### Deployment Order + +1. Register the STACKIT Project platform, so tenants have a STACKIT project to order into. +2. Register the **STACKIT Kubernetes Cluster** building block definition from + [`meshstack_integration.tf`](meshstack_integration.tf), setting the STACKIT identity, the ACME + contact address and the DNS parent zone. +3. An application team orders a cluster on its STACKIT project. The order creates the cluster, its + ingress and the Kubernetes platform with its namespace landing zones. +4. Register the starterkit and connector building block definitions against that platform. +5. Application teams order the starterkit and receive namespaces, a repository and a pipeline. ## Shared Responsibilities | Responsibility | Platform Team | Application Team | |----------------------------------------------------------| --- | --- | -| Provision and manage SKE cluster | βœ… | ❌ | +| Provide the STACKIT identity the cluster building block runs as | βœ… | ❌ | +| Own the parent DNS zone and delegate a subzone per cluster | βœ… | ❌ | | Configure STACKIT Git (Forgejo) organization | βœ… | ❌ | | Manage Harbor project in global registry and credentials | βœ… | ❌ | | Register and maintain building block definitions | βœ… | ❌ | -| Manage STACKIT DNS zone for app hostnames | βœ… | ❌ | +| Order an SKE cluster and choose its ingress exposure | ❌ | βœ… | | Order starterkit from the self-service catalog | ❌ | βœ… | | Develop and maintain application source code | ❌ | βœ… | | Manage Kubernetes resources inside namespaces | ❌ | βœ… | diff --git a/reference-architectures/stackit-kubernetes/buildingblock/README.md b/reference-architectures/stackit-kubernetes/buildingblock/README.md new file mode 100644 index 00000000..db6466fa --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/README.md @@ -0,0 +1,36 @@ +--- +name: STACKIT Kubernetes Cluster +supportedPlatforms: + - stackit +description: Creates an SKE cluster in a tenant's STACKIT project, installs cert-manager and the HAProxy ingress controller on it, and registers it in meshStack as a Kubernetes platform with namespace landing zones. +--- + +This building block composes three Hub modules into one orderable unit: + +- [`modules/stackit/ske`](../../../modules/stackit/ske) creates the SKE cluster and its kubeconfig. +- [`modules/kubernetes/platform`](../../../modules/kubernetes/platform) registers the cluster in + meshStack as a platform of type `kubernetes` and creates the namespace landing zones. +- [`modules/kubernetes/ingress`](../../../modules/kubernetes/ingress) installs cert-manager, the + HAProxy ingress controller and a Let's Encrypt ClusterIssuer, and issues the wildcard certificate + when a DNS subzone is delegated to the cluster. + +The three modules are sourced by Git URL and pinned with `?ref=${var.hub.git_ref}`, so one variable +moves the whole composition to another Hub release. + +It is a `TENANT_LEVEL` building block ordered against the STACKIT Project platform, so the cluster +lands in the meshTenant's own STACKIT project. The application team decides two things, the cluster +name and the ingress exposure. Everything else β€” the ACME contact, the Let's Encrypt endpoint, the +chart versions, the issuer name and the ingress class β€” is a landing-zone concern and arrives as a +static input or stays at the module default. + +DNS is the one part that is not complete. [`dns.tf`](dns.tf) holds the whole delegated-subzone +design, including the assumption it rests on and the `modules/stackit/dns` module it still needs. +Read that file before you change anything about hostnames or certificates. + +The user-facing readme is maintained inline in the `readme` field of the +`meshstack_building_block_definition` in +[`../meshstack_integration.tf`](../meshstack_integration.tf). + +There is no generated terraform-docs section here. terraform-docs cannot parse a module whose +`source` interpolates a variable, and every sibling module is sourced with `?ref=${var.hub.git_ref}`. +Read [`variables.tf`](variables.tf) and [`outputs.tf`](outputs.tf) directly. diff --git a/reference-architectures/stackit-kubernetes/buildingblock/dns.tf b/reference-architectures/stackit-kubernetes/buildingblock/dns.tf new file mode 100644 index 00000000..78f7aba7 --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/dns.tf @@ -0,0 +1,66 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Delegated DNS subzone +# +# This file holds the whole DNS design of the architecture. Everything the cluster and the ingress +# controller know about DNS comes from the two locals at the bottom, so the design can be replaced +# here without touching the rest of the composition. +# +# The landing zone owns one parent zone, for example `likvid.stackit.run`, in the platform team's +# own STACKIT project. This building block never creates that zone. Each ordered cluster instead +# gets a delegated subzone named after the cluster, for example `cluster1.likvid.stackit.run`, and +# that subzone lives in the tenant's own STACKIT project. Four steps make the delegation work: +# +# 1. An NS record for the cluster's label is created in the parent zone, pointing at +# `ns1.stackit.cloud` and `ns2.stackit.zone`. +# 2. A zone with the delegated name is created in the tenant's STACKIT project. +# 3. The SKE managed ExternalDNS extension is enabled on the cluster with the delegated name as +# its zone filter, so the control plane writes the records instead of Terraform. +# 4. The cert-manager DNS-01 solver receives a STACKIT service account key scoped to the tenant's +# own project, so no cross-project credential exists anywhere. +# +# The shape follows from STACKIT DNS being project-scoped end to end: `stackit_dns_record_set` +# carries its own `project_id` and that project must own the `zone_id`, and the SKE +# `extensions.dns` block has no field for a foreign project or a foreign credential. +# +# ── UNVERIFIED ASSUMPTION ──────────────────────────────────────────────────── +# +# STACKIT documents free `stackit.run` subdomains as one label deep, and it separately documents +# creating a subzone in a different project through NS delegation. Whether delegation bypasses the +# one-label rule *under `stackit.run`* is not confirmed. It is being tested separately. If it turns +# out that it does not, the fallback is a platform-owned zone: the parent zone stays in the platform +# team's project, each cluster gets records in it rather than a subzone of its own, and the DNS-01 +# credential becomes a cross-project one. That fallback changes this file and nothing else. +# +# ── GAP: steps 1 and 2 are not implemented ─────────────────────────────────── +# +# A reference architecture composes hub modules and never declares cloud provider resources +# directly, and the hub has no STACKIT DNS module today. `modules/stackit/dns` is a required new +# module. It has to create the delegated zone in the tenant's project, the NS record in the parent +# zone, and the service account key the DNS-01 solver uses. Until that module exists, a platform +# team that wants the wildcard certificate creates the zone and the key by hand and passes the key +# in through `dns_service_account_key`. +# ───────────────────────────────────────────────────────────────────────────── + +locals { + # Empty parent zone means the architecture runs without DNS: no ExternalDNS extension, no + # wildcard certificate, and cert-manager falls back to per-hostname HTTP-01 issuance. + dns_enabled = var.dns_parent_zone_name != "" + + dns_delegated_zone_name = local.dns_enabled ? "${var.cluster_name}.${var.dns_parent_zone_name}" : null + + # Step 3 β€” the SKE control plane writes records into the delegated subzone. + ske_dns_extension = local.dns_enabled ? { + enabled = true + zones = [local.dns_delegated_zone_name] + } : null + + # Step 4 β€” cert-manager solves the ACME challenge in the tenant's own STACKIT project and gets + # one wildcard certificate for the whole delegated subzone. + ingress_dns01 = local.dns_enabled && var.dns_service_account_key != "" ? { + zone_name = local.dns_delegated_zone_name + stackit = { + project_id = var.stackit_project_id + service_account_key = var.dns_service_account_key + } + } : null +} diff --git a/reference-architectures/stackit-kubernetes/buildingblock/logo.png b/reference-architectures/stackit-kubernetes/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..57c2c04b90ebe5566b5172f3f068df442eeb4f7b GIT binary patch literal 22760 zcmb@ugO` z-lUYqnt8rNaEu3&a+W$vXZDkc=|MMy9QY!QPsMm*IL43BK>#j1ul6t@mPxZ%``!)k zR(B~HP10=`*ln2pTo5OFZAvw(DNb+e2kXg8@UEgGJWLeT19uj~aYan@hybRwCRow;Ggo?O?#kGFNgngoBe6MiZEe9ahU<;lefei)Dfi7a6z>|~RE zT-KZorvyLREBPiJFV`qAMihLLw7X}t@i1#PoEp55R+KjyNvKo$De-^T#^_U32)-64 z{G!&>`Wh(kITUSIe(D;z;K?Nk_AUsE#=2urvNR;YVAJ5t##*spj6N(|eX8hGX`tr< zXE<99i*zCh))M-EFGIKcC4nJXs;em=$=$VM1ANh1yB!puCLl2R?h{e~58eTMuX)kW z0E&CER~Iy)O`rEDgU|U-mPh~qWf*o&HUs38u0_s<(}VAXSq|cR61KMK0hHPQ^};z( zUO_Vv{Rs1k)411DI|jRY7p0jUdP_OWFDcaTy_`g3Xqs-;^Cwi)&4ir*z>Vp|9&Fgn z$_pEWVrf%X#`5yG-sVwZbZ8&9=Cl(Ioz^jyuv$_m0Q=(dU)cK+<)?A6CUFopjp9^3 z)b-=`%voSDXxESi0De|R%6yWW%9I-m56y7$afn~YY@wQ=l=NgdUz=MxA zb=~CPl5huZ%G%kGcYQcjMN#o}%vZ(-T#!lB(%>N10=Wi8#>{TT%cp z&Ar(5_Ym~|PlZq7#*MHLaP-^e1?P`Tbnm8Nzw9Y0atsEh5&^*?O`}0`PA%`-v%`lL z4!AL&tXA5#L1p?pMedH*_k^^Ee!uxTTx}}5X*}q|#RITqGe7M4Bg(Nzx!NzS3_J}s zVfH8^i&lygThIvXOTXB~#*VNH%3VuVE*3HpLJS1LiGeQ*V!NM~VH(A<-3hStx<=)4 zeptnASesf_Ky)Y$%2@hN*l#DH*KWwHKS|*9fR!z@gD2b|c{f{o^%g16OD?fhUWB?S?y?9|38p`FnioBLE zWedkF&KeU|uhDVK1DqOJTq}bIjSOfgHBY^;M5vOb9{!iucT?__<>}aK*rI=qDM#$9~3Yw*7IVY8-EtkER0`pyB7)VNQ50+R*5UlF~I18Xq6#5<-xvB@fc9=Y7Vx9PV*3c-ZX8U>$1|g3efa_4Y?Y%z z!!s8E$XG2QqS5pTBZ+{X3F^KG7?Nv(sszq2`<#xTqLtcY9yH309@}kAsrNjnSBw~8yQV(Mvk6pmIrq#_J$8iXut`y&|FV_O1Lq)Z z+&yA&Q2};Pzl7Ppy{#>p+?LUFh+Yubqmud@7ni4(jE#J+zx-(szOt!s|guwxzH-7$G zICQw_hKNQnTe0K8yD%8xje+gB>)=w9RTX6JW&Yn~UpNPrY@63L| zL3LJWnq4WS+8d9WhKT9xh@%Jm{UR-M=NAcCGwdWAH6sd1(~U&HcRgkY4vh-Y-&=fC z$N|)Ix9V^e<#navjy6KZAFzRQPv0Pt+XD)3AQr~6v@x%3WcQ8S%@PtawbH~VXK4KE?e+BAZU z{ehgsvKQ#9tisTLvh~Z4{449HvnH`DL#*&^fyzi+N4C_Sx;PBoa6+ccF zP{H8gX1%WCs=LjIubA7G=9X&m&+M7sQE2<*qMkDTw<)nYwmuFLh%w?i6fQIvOW_1F zE1|7d-YP44g%M!^(W*<&dP!PuK6$WSJ~ef7HxbRoH+o2C+(mnw=PbC)28_Xy#1*5}^O7RrmsL&TkfjX*3Cuzj&;RsUi_t)xOmT zg+F+K+*P=xt!q)9{ny1TXi{(fd@8PxbZKtjE+zWs5ViiHER6VC!DHZpc5XcYe!eVrh^=I)lYc{0$$ z^5w5kWJ`O!uC`r$MgNn6{utEnqrDIp3U>qet;&;Um9-1>k1V-RmkYG~md@iJ1B#r6aQ zhWm1HM55R@%#?D}{P>cz-Xhl^UH8IzNH@6s_kg2uzezE>FD8J|gZjeD0h<+@-w2Zh zz9blQFiM&IKr1OGahXEYijx^-i;5<8wO89kE_-ztR@?N-=b5DK1IU2uU!D@}LnLX4 zy$1}bnPigxmdtKQoGF)EBlXJX=rO{(hIGK{PZ4ih*qd?=(v3hGjE1Xv`fUELrcrbs zRmg7{{w5CP=v9R|AlA!I_9;)Yt>A%k&noxNZ}(wn_EPndZ_Q>&SeN9zIf^}xYzLsb zN~Cf&O785D{{zPfZf-l;5MfjaWmboR(Eu3Jx)WTht%~f!U1({6AyL1H`@4MOmqXet z2zB^UkFxc;o~ydq?h&_QHyNFI(Prn(vFufaejrw@DI9)mrze#VyZ(gJs5mlS))nK_ z+g~;Q*~NP-ZKyH~0H%u#1-S6*aE8U6875h~JU5xuu-)vv5*Dzb9wd2L#r*q=BVxtz zYh#h}A^h^!hWv|?#bLBs+X2R7>?&}B=e!MdedK^=BTNK%r8}|AV|x}^JWD~OET28w zh;D?gM%iCC(&Hdm)z~$p0^lx?t+n4j7YwDf9PHCwk`hdmEtY>ur0E#-E_h_?0o{z1 zLtzs3l(hr@GJ*-}tG}HB@@07hJc2BX0J{ymihvNhQ@+nX@NY;Td~zyD zPULLF1JiQ@+{9Kbys$-}hB2&|;{b$5zG*u(FeIE=Mu-eG?=iwnq*`6hlo@rIvXT0t z5kgyX#+fgOehEC7Ii~Gi>;>SmCnY4ocU}hWnQk|B48HPW;2tcxb2u6nkr|@}7d=^e zlv}lS!8aOk`ELFL1ID;?qn7^aiEkSQ7g$nAY^}u0x%$6@g%m_t&p#ZMr_Q+TzL`|p$}RAiRND~Soyp@uQt3lFVS9RJSLt$m<;v01cvS_&)Y)bJ(ndMG5t!x5W?e%|Illw zXmGXmsroY zDEm{l1ddGhz|ob~*h}`~1v81_^-a^AW2WTHlXb9HKPH>;S{`AsHcYY=-&k2kCbio7 ztcndeNMBW(@dNk%IqsF$ODDRf+nUsEfL&QPm1$$^A&YP}6G;~=JHkA|2fFRpza(Ti zSaHz;-9ov--ykBn164s8O%T5zaw0I~Y%!fz%;)qS@0oa>Qdf5t>mwHxc7RxZWxBeI z;==TkAMD@gh*7$~)tKq@_Mc_t+}V6C7b<5wX!+Ue83M!2)&PS7udbYo?sp3(k%YWW z!vS@%R7MAHQ+ndk(F4lsKV*78=oKQ6kcF3b&u8l5c41lPt*gl99-3ePV9&svG;KZ^ z>|f(n6zi%kSANRxOEVF-B|Ik8Z8-RV6@Owa14pZ`d2ae({J6-{5mv?8jYvAcEmQP& zgf6qAvxhJ)&D{ONc}wd`GU2{u)mwQu-aQQExhONYNT^_oBQ{J=& zXIf|jQ3NI>VE7>y>v&fgDeXpc&_T_`&aR94p?0y3CllBKWwFLlS%Y1TO`)H6%Cgn> zHi>{mopmU_Y*fvgf~fwPv;d4&qauYFsUo_WNB0&ST&#$$XKs?EBOI;VoK2faMD$|9 zzjJ2#o_ZJ6RqAeDAj>#?o+vO{UcwaXRY0?dYoZigl z6}4QN;TzSaV-CX0w<@!AGRX$%c%*a=whlFF)&sHiQ-tUNULRYAoa{e*($@sbBVg!i zdBr;8qv@{9OMF$&w=!61tY2rvQz%luh!4|9l%vDfr24IsxGK1O$CdGYFlxS_{!8)R z=011M$#?k@M90MD2U)R#$ja!=UNf7f^XQZcq)1`17O9X@-Lu$xzAb3WI3e=MuW z=fX2TA=hNXTC(<~7;a4#KKk&pR(u=l_Kr@;3R)EkF4K%yrn_`NMedyuPgW;d;-i0c zbu`2Q2r=};z718LqtLZ%9_Xy95_7!>?p~qU#h;_~Eo+(ecRLiMNQl|p676!&Giqef z^*Bg#au;CU`eoW3Q2djk>gbL?8c5536#?ptTPEvWvDDdcl{YIcDbF=%bSw9+kgC3!tu7-)9B@IL0Zm$Bs8~tWeY8Q^Dc!Ud*?p zapdt<9)Sm%mil~n;cO)2CUfX|stJweUY{T;;c5~}ZODz85z2KN^7oxM%)-#GcN&17 zFD2_WS2(J%r?7&c@!1`NWo@7&PxpP*_#sjMG=WL|rZ?Smh(CR5_f;t7U zr_dXmaq~d7i=NGvrdA|Rwm{{n#PT8w(&6A)@}uECO5n!kD?72qHl5X88e&w<=zvAP z^!+(sDMYR-oO?*)*U)%vd-?W(=7_2~GGR>=r(Et}Ot`FpJH)rHhz2EMpAiM8>bGw5 zo5EQV#pQnpN1ZgRTi!6t4nK|bFQXt@GC;Ft4qFp~%A$(ASv-oxEri0F#+QK{$jK5h z0D+MoGwLn*8|}tt+<0cpHnJ*}l(gc`imjt3PV0%OC^|o;x8d3bbvLe!^JG$$ZqeFC`eP3kb*k!U_+f_p@f zgdwW!>$jKnz7+D;7p%sdC$a!2BM^UlhO3RMH0?U~QuL=FvOE5`IqrWJ>0{u&kds_e zxbC?fhJ4GR$5-h_+}Jl?n(?}*&Wjv~6n>Rh`#ZhMtIIE#A64co14GJ)6wVVNhV^9p zE($_u%?^@vQ9Y63!|$eFTSed5?I8jvhY~PO{I}**pfN{c-^S^Po4iFTtuDEuXLchu zljn9=5xI|JL6>2xX57zum+tfBpR2(sbq^WDr>eup)07l#f7A%k`^4$Vx}2KWd*mm! zSDM`H=CzHRzxQ`Q+ArlopD6f!Yq>K)1zto!)cky<-0s4Y)y~X)idK>wz#8#Dbxl`G zl)QI!vE(PY#KF#BtadCXiWJ0!T)1WCP{%{8&brSwv0YB0SVz|ln;owweq{ZI(7bP8 zW|Tw@q?hDPC|seQk`c?APZ>W`cG+2Y(hx$e6S9o-U%F`l3(XZT_(^(wEr;DJCe`e;}AI! zlp!WYMe5g2Yl>ndc2&H*8TP}GI@V(Cvj;O!uNDNNQ8J~fPAR@60vv7*y9sd33wn|( zqv_Li*daAVea{EC74i?$RcB@PzK>jIRR@+;`vB0#&W2L~l9Up^{sD^yV5{+5mVK51 z77g* zT`GB6zYagGjMl=^?skv?Y#{!0SfZ(~SAPD*9MK^^`i&lRmc zBIO33I}sqLl}%EYmks+2JwL~1(=T79W#lCfPu~Py)Y2)6`eLrjk^o*DpuRg}g4!)# zS;RE$KisYC5mKIeVGp>8U+IDNJh?IPt3gZfg?=WZDhOsio9KwCxN3=aG<$ot45P5C zay1mc-&zJG`g0BRV?sekcl<-V6xx4)T{8@o`TGxeaqoXkWd#iU0|4FN`4RugEM`@< zGwN?EOX_T(KcwHS{CiW`_*tWg{MkcLR?*b?L43#BUD*-j($;pTgvb+C&JMhnbiQ(y z)4HW`r6-p#Fu7P_`ZQjP^~@Gr0gsXU$=hL9a!)-F7H3=Jf_85Ilj(tjGFK6qVr=Xp z6~BQMp@-o=$azUn8mlWS*fW+vCWbpr4q3;)g>qT6;}d1_^Je}7t>eu!SuAzdb8G&R zw-uLSy)ttj>Z;Z>O7C0-tLAz;(uX19AmQ<`mGQA6W?rI~-qeVf{d%*V_*z*NX1Y3Y zKtHm9o&|7Xa7ax2nkU8KF$bn<{Gii;cUGW!K%ZyUf4Sk8_tm8%%}DIHY;RGtbcz!C zZKdr_#xRH*+5_L;{ckR-dp;!+{hZ&9S!-RM&|sE991XXmxp4N+Xja^}KuCcv)jECI z?Kju(6|`mViYHUQ1B-zd@8=5!2Y?qob?S%btWo95^Jxu@^#7AxDjuy!|2BLQ8?|M) z4x+c;g}qv)6pXvWwf&3Jdx9eCE~f)CYO_^EaF%zzH!) zurr7R24{lV)q7zSp~l@AA-*LNOb<Ai)Zb_o&8c*Vo&Ewq0+IWqq3B;+v*h9`xRRNK71j z0!Ql|TYq@tkK47rY)?x(ol}#QjY;Tppl?0DEjC1T1E8s#OlXbenP#nBKL96P^`QQ4 zCJc|>GR2G%(jM!r(xej+DVEq|+BnYqR~}R9;^)DJaQBiwm-uaDJac~Nko3N{sQ=Z7 z#~XS4OUvQvw^7k6poLD} z?+VydH#kGzZ(Br$$6i$%Zsge&j|>M%R@8 zW7+!2S2nUQL98_SQN(pstfu0F zOVX>uW(Eg3%BgfPCZmLNvwnCge3EYj7qfy6b5NgU4FgAd%2S64+?T1nMV!d4HTcw@ zDa0bKqA(uZlYyHjp5%?FN+)(i0rQpfTQM+S;C`gMy8&R+-1PfGd>+?QtJAhQv}!! zj$Qu&`91p~pzEQ=f9T=zo#e`UBvrl32tzQ;&d4{*T^qC!YpCcvGkLThxR=uv9Bb3> zlFlwqE{qyJj;>$79FL&~o(JzDHWkuXJ!s%nP!PPwjP7-dyv-G@wJHK@lLi-h{`4~m z@7xsphrRfB;ev$~wG~!k=?kupu7a-Q2LEU8XxEa&6{29te|u{U309IeWR%jWp|@;< zTB*G@$0vlK1)^xdfh5`+Gdgi&=9ntz1Zd_vX6)!D@6fMFU|Ax$5X14EGoQUCYaMVT*IU|}5GaB}Kyag;~-w&cI-6l|74 zjVj%N4FLaWxaH8BJGE8>bv9dckED{g?mMfk|BtYD`*vK?^sn#bxIAoS>Gi&s11an+ z>zmE8_zw;9R*0F-K&w6mG=hUD1~OZwxv;ZEgK(v{D$zlBtC8hkllt{BXFvJcuPqcL zeqNv8SgcZkF}b8d*C=zP1E8T}|Hq%4DKl*_5S?LSg_I>*%E5?TF1@%Ve{NH*Am-Ot zXxUT(2~B!Eo4lg<;_rn#IaqJ|!(N{%VzatuLk1`cWLP$fQA$V2zpRA_J4T}v0tXK9 zt=+CS%>E#8*$=uVYvOKSYOi8E~bKPA6Zy|HN5K-0MobZ z$U6+^xa)&gTO-E-sP|@ZPtfLEe`Z~}Fb3@nyu@C2H zep~W3W`K!#E|Y`3=;N7zmlH1c@du?TyYN`#K4IZKc+pGmvRUTi(RZsA*6A-#9{RlB z%FTl2jLP`>% z{S6F@u$12j=aZJ;P`{AYYo6iyEVEV=0AJ@Fo|+`Z>upGy`!U`%YtXpZKq#gjy>*h) zw10!@RCnR0#?}K=p-e`$6cp$q3*}xUa`iP13ye@ znpoP|ovE$v)X~~w$)szbQ?lJ zMuGG3uSKzOd)4YqMJm8HXHU&nwX%|MsiXJjk3lEI17n@UJi@q@K1(pQuswUU;NIVn2hKPH!l_+%f zjZ|DWZ(@4QIL^PyADwMcc}D{yzU4{8pCvaF(w=E#-hU-(@8=OSNA&Y^RT{NE&*0mj zeuQ@XY^5T-Aoh<8+-@FUW=hAr8`?IJ;eA9Z=xCW6i4Wx+Ec7rv6hoP>PvB7V1#Libgj*<*4?GY!x#LBGxlwi&m@D+G`c- zAZtcnG=a!#L)W)OlSC36N@T5bKXE${YdTuf`%5OA>(GrRJBP1 z`q@QydY97(NgEss@%SLIT)BTD%5qb`Eo+#lr9WSMAy%9QiA1q4IKSS(ptuaBxs~wnJ0WO-CC<+TMw9X zotiitGdNqs3^FVtrL?5kXG6r>J4w_TgS?S10>NC{fS@=+2~m+kxcyD=;CLbbQ_ zTujvQ%-;>N{(|!n{9ga|4aO;fQlqXXoD;Y$iE(3zu2_-KQtdj$o`jOwh<2$RpV5aC z1FLd@NdO&{gzM$)m~h zkgJceiq-$B!~Ht6!+-0kdjOZ&;q6BC}Yz5q@hlilToA6cORRW(*bO*o>T{vbB4 z(S4UD!a2b+=5v3vi$efa*R&2udei5WVgICTC#r* zI-m($m+#*V6>+mGE1lAiEj^^lGew2F#&cCn-f|TDXw7t0%0I70m9Tt{_?{}9sD-_ucJS`4LNkqEnH)Af zG;)gn3%_5uYF6Phv%zHZ#}f;!`1a(&S3w@n)xQ26wo{T zB!80vX!89gF*otgf3{UPY_LVnkTO7BL@7Q4D~eL>rGnvOo&OO0b1yFJLV{X$)J>i+ z=-P>WGo^(F%(UDpp@wyq5xYHshzrH|Fns^i(oPw{m0FbdR-sxhy-IX5=o)ArdKDwh zh8Tr|e+Qu>+K=D=THu4WMC0BF_#fOs4v~~N_G454yN}aiDFZGo5$-}DrWnCtSRi+M z0@Tsr4Y}dNybAu4llDb4;)ld5CXg%@XDyBd&%5B^{o<=s@4=|@NQ`hxf3%F0gA~ z_uH9kf!5h!=-_ZBYI@B?x!-5e&(iwotQ*=80gx@7q0~L6s5$b&uCb=}BzwKAYj#p5 zr!?MFY#;sc4g_UqTGkd41`Cehy?g&aA~yxK%?I2%yN^V$)q$l20Mg$jnk*FPWH1uq!#O;0H6F>rhXS zgYfKdVK;N1QO>Rkx^=K|_fzA`&G!wK{ombk<4>KKbXhJr&^>0w6eKKCpY^p+N$JWzVfk={cC;t3mY<8E}}+R}~Fv+YM_Jn_ED zFzR3HVc4JcE)<@_!{jQDzI?#T%wAX46KA_yaM3_8=-XD`=9Ns-%<;Vo^uVZ4y9F+H zdtLhxTi$xJ8H>^Z!ij^X7J&feSun{&cessaY2fA7Eu^l}Q56Xpd0fObtAfj@1??Tb zQ1;T;qIk%JbgYvh&6=%KMksscVxv=GZ`v!il#K}m<^H$(X{PFl5(`BSk8W1QoT-^i z+s%ixzw6d_y8oAaYwJXNbvl$CV^pi6T^=9hVFjuHh8I{mgloR5##L&aQmPpY>*R3@ z(fZNDp#dWy%I~_p`*;!hTWqC7+wuCziJQP8d#^bDQl>{hMpOKzU1sCgOwQzplptZ&PAh|CXJuzXDT0xv7*5_zuYX@D8M#QyMgs zDy;cos66qxN#fwJ($4*2n0lM6!z18Z#r;cMPTYZgM^~r(pH_CO`!BTuE>NlNgjCOE z^>xutR?nJvv(9$Ye2>wEzMUVU1je1F*^%mmLw`c!85DZfv+bfwA>L=<}#Br0qj3HbK z75raOpkdR0ZQ5QhV^k|Gy6s~CDZ5wo5*2F-i33A_xtN@Z?k+fZTr57kVo8ZRdgyjqWCk*GfFrKs1_MABPoymg(WMS(~*z*HA?Oaz8{7oya~Hj z!O_!7+nScWq4T8!=|P>36a!hTV$YgIMOhEi$5U<4XMLoepJ&&lfBD9tKd{G5VritW zxcVR!7}wLt!9!K#nXRyH1n!3dvi4?|7KSfmHa;jR_uC=qo}1vpQyt}nlupGl`L=Y- zA>LgsUQq}1;W1IJ0nSZlV;L0v?(8u0HsQ9VF{x!P8ZO5HqLY)M+@TAN(_t86a|KU0 z$Q0&=NxmvOEn8>8OPE-~ysq>6RwM_!V1kTTJ~x&nWD(v^^v3?qV(xT@P__T9lKE@J z+F}?U6-AJ?eQJvD-Rm9+nKyj>r-5R=8kF)Mr zbMNzHo!jNDH!9=XX^2&GuNeAxe^;{uB=pCYO8pybez5#f{q45#(}q-CXeFf@!fs8n zJjxXw_&7l4=+VW=*gyllHd*d_CX%t%ZO!H#k`S^r`M0()Fpq3y$VP>KM3&7NFC621 zyw|uh4EaHcpqKODK3htl<2!rWWl8ek_ZY18M$Gf?zva$9jI$w~Y-hKoA`aS4=wA{z zXRa@;j23}qjtY2pfnMqk6E{i=qvNrsXiVRz!@GTB9lFry1JqfnGut<4y?%aQGkRyiYCR%$;yc)(VW%u| z{sOCx2II-mzR?h`rRx?0)8z(ZuWaBG zqBcB~UL;^X@C5?WG${sJ=NKIB8h48`$%ZSf(CeqNhG#f$s(6paH13Q8BfBQggGgyj4`Po6` zJp+&03udwR>a$E+4|`rfoxYM=Oc`7UYtEKxglMCWdbW{ggGz!$K`bs<}t~ zAo~gd*V%3v3rl1U-8xG+iC($r_>;3Qi#j0FUQXcT=m@eH8nEFQ3uB3x!7Dg&cHNzoa@o|s71*w`#dD6W; zl6C0L2;0@We$=6hQlDdNhH=;bC9UjNM4m+dQO1mNgJ3Rrup>FbyW?COJ?NlNZI7<` z-a?Jc+7PPLs;fk({OTl!`Uy>sRq8}exXjnR=WHmkA6YvsH7Wda(a;buG%?|Qy7haP zwL29IPN`PrvpowzPQ1B=u|?$Zu?U{ZjW`pWLteSAi+WyJ30A`$c(!au1NDBo8FcjcK*Z*Jg~BtZHON{hnB@=i{f1! z+N>|3i3YLycBzQuBA17R-G2)y+3$NnJ@pk+sQKjV~?D8Xf7vrH3pS@QUq@iYj z;ECz-Ch@qroXu{>ixT{*s8;a5m}k@*%$3VK?U$44{pH2RG5$zy8a7qa!E8N zRVN_XG096Nh$*wCAQkhoP!&%S+#8~4@{WOp3x9`J5;Y*MIaZgC3Zd~Wn{Z4{a}m0k zJdhdiUU1(3C&P#%Cq_+deRl6McYA^>^%^9B>*Z~UQwG@asu<}<_CJQNm0#UDN@`O& zB1Qa*>Z_|ZG->^75gFXGC9)8OWQ?{((v>jBycBsk`FK!oqIK*nl86`f_aYRX*jpKN zH(EB1i;RwMX3~FIdV8GWpqWjo6L||ye1X*{Xb?-hpnCwG`k107=7l|hfg6nkv;;?J zvO8pIZtK=cx@CH_OVGYT;9c_*G~3$rVw-}{M^P;-XXxksSBr{I6u%&J3aAn<(@@;F zn?A`%nB2Y5K2nOwxR;05J1?O3z94PgkRP~;f}rnK-gdBDE(Z%vFgQrVF##fM_puyD za7;%*Mg;aMvV^XdH8|2d+8txr`>ih5H~;93G?oh&HoB+nmJ&-n8A+PQu2O$z*QSYU zdhsgOMXl=;FR*xEeQF?$ z6P0qdc$_yU)~UsBL|`|K_+|$CX?0iDQV`?5YuSnLqkO|D*V~cbS^7~uxu6v~D~uHa=XL*U)ai$2l;2$5Kxrv5y5;LLF82L95n z_?vW94M)X1VJ;-^!yCuGyk5>@v(M+l5*eLb1zhokl_j2!D2RgICSosc+=x)NOUmOJ zwdFdy)AaG0mrxzwfVIW~+G~qmCd{!Ffi)CMQ?CQ;9f7heQPy-oM zHmjE86uubmzn#{;Wzcj%cBflCz8HyP;_ykYYuIa3i|RgZe1EJy`(3Zb|3agKN2Zdl zeW-udnEWe|HU#xRJ1nm;6Ui)_O31$XtsB1GN%tdx6R6$vzE0h~=OK4KL2LGXfa}pY zX@FyQ0{i>myGE?j@H`4ShED2jW~!@${k>=BHVKxm)4b+-m_xhzDm_fD_z>VRL&`G~M$Ah_G%ce;HAz^r zb1t+xcBD=;5hr4-b#OW0UjUH~$iBO|H{-ftExWnN|LoZ|MTLFz%4y%Q*d!O}DQzZG zYY`8L@aj#z(J#j=#M19utWA2~fdph-Gi$I&AKKJtWHhFYDk<0wn-3sA%RAG_YUv4h zj5o#F&EMQ53g&#puLe`5YPXk6m@~c;zKef;zNzW2G-K} zt7DMn6UxIAgh16i?vf@ydSMFi1Z9aqURn`-g_cN)k zlQe#=q+C9~JJEEx&SIAQ+y%d|Nr*Wk&`A<1{vi(;52cP4{voPcnFO?%Zl`8BbbGDs zK{{hZzo%{-HzVH2d5tyCN^|CPYKC0QLVsftg7*ph-c~B)AT2XoI1-uzGSiKVBelf$ zR*lyQ?{V4eP7Pf11u z0V(j~F)YZpzWnRB(8uK`)JbmZ9>Lx>r4c_GKis7w-qk>HQUo*e*MAw=P5SAj68(MZ zsA!^^snPS1V)#+O>vwZ&ir|a2QI-_JU%;}IX4Z4pl2fTvWHKqVW#Z|BkrG{V4$QI05EkTo(J*<&K?)%DSO6Br zm0o+H+&0sme<|@g0b+5QFuP&ruXI`(^$LrBY2#7fUnwe?ZP5D9it^nX?)wj`I7J!~a6Tdn7dD5iGyYUVl}o>kJV}8sLeOoy+nRV^)QCGL zE&B=k|3s9+Kyl`=e(GWaYVIDMJVDq4$WUL@NEYYC?ARvqY&Cz@oaVTM-N3X2HPWy29FXwLOA=0K1X}G_=kL z4AvYnD!fXNc>Ch46c#FH`rxbg1ldjV9x`ly-xcxjVik^*czb<+W6=X+_{RswhH zzZ>QR)lvZ8RorJI-nn^CM-+E8B(KPIkSf-{a|Ao+&=p;O!2BYqv5N?3;3PmZM+y*o zzy80wZ^iGhop7EFyU*H_1Li1ogsv>_8qoo9 z9pJ4<0|an!9Z0KnFRGycf*K-lAb!3HzI9oO`QIh>;Qf-#O3PKZ+rz*XWzUB1Z$ioT z^TiuSXn}^`^jrV*Ko6G5vTlJN8LVcN4>bc7>(iW=1-!J{eMPX&^6Rlntu;x+32DqsaepzWja9pOJU+Y>|^_r7n*km9|-`L+r4ZJ^lUPVAH&tZ$^*V1|4%9->2>_3 zq)ULh$BU`O6yP+Oh@=seQjCTR%@GB2wGCtj0m0gV_<4I*;O!t=wQM}eU-B-}pC9b3 zPzp{P!OevOTK>TI2$0Rc`;Zi1N8f^Joc(uuoFJ|T1UMcT>~?&hOKAu&ZXyB1uYHVO z(Nh6_{)!pU6mQ&?e&N=HvMYNjx>;_Ky2PYK z@K3Nlv0)~GYCn-Ro1Z%A`BzLV3zmH^ttcn{rmOr9MA9ho!@hw z<@-G6YmQjRxuDBm`R~e?Z5**eb8*bpdnSiw!(n;dU@0XG`mH&?&FB%p1J( zD&xG6#@ok3w)-X!l%G^zabU{od}Em0`f)XGAd~e;f-l|T5dP#RVG>3UuOb8n){N=d ze_oKt$*1n??Hs_$`yJ3z^v+YwJw zUAd7rt2~NR)^ z>a;lc^h}~-LcZWbc&r5J{>@getPDEP5XX)=*@5%PNKs#{M+2z#d;HV9U83A6D`^?z z=n7tfjlrZHzf5^?FmPzXTWI8;0xg4%*UIyT#qmFPK3Tm6zn#^er2CvwHpZUJ zz=uBpVcx@t?!>?a$<;LU;z#OVzOtYxkpD6*i3Rzou3iu!{Zf_9Wv;!+2CRnGmt}-| zKT{@V`Xd5b79Wa%(HXwhez0!{Ra2gSdFQ9LEIjOnCe7AU*%AF~dfd6|Hi1Lh?p2n9@R;Zbw#7xR_&0;+FZxUL_`b^lwP=j^o$r79 zoq?O+v$Q037o@uK!}WJ%El?mIo(`TF`S)im%agPOk6P`VgpZn>oHXSJlgy7a5o^cc z6VsN6+s48Ct&k8VeLXL9{!mi01W5&Q!`#=d7P6NXh{^!ZMin9Ng=8{txgkNCw%vvT ze?Acr2QBsbls9R>G?KBosQI=8X<55o-x}B}s30D-iCA-1mWJPn%7eKG)p!!nI7u)x zXv=@l<3VDBnt zne20_?iyu65{gpMM@`I;Zs4?k*#i!jcKj*t^FeC3{3&>`KsL>4@P2Ff&@k&%S-Y(RT)*UkX)Tp4V`fG;~MjW zb0}aUkOOOfM}y}X%n-tGUkcp(x38|vvfgHcbDw`1`XQ6H{KerNO@ZGX?Gaj_AOEA| z^&Tq(#ys6j9YJ#BXhPwu5TWPmNP>H$?q;PqfdbN=0!vYl;`%_FUU}aXMoZaGt(4yk zQ-S5;;p$yOFj&Ekj(m;F=5BGn$0Ut9$9@8zm?r*6Ui@=ts-1h!zqH zrUH))J@aCNqp#KzqGM#TS>j78>y=N@SpBSDHV zrL;$|V#|D)VMqBMRPl2%}*m1VT7GCy4q&(VDS~t@hndk zAQf$$3pq2il%6Xebog1KX1hFqZIXyJGM9ssf(alggD!~+S*mbxyw1;6KK>M;fBryB}`my3C+zNo*q(WtCP z&|3!_%|d}KFim~EOw__hAi(3*(Rg1}ZjG4K9YF;c3_aHCk24~aN27$}8<@7r*zAk)Dm2Za_nF?dWhx z(nHvI*i-f)j<2;B3J|VLVa$X{>nhOt;hKHygo{)Rp7>UncGDPTxu-ci3P4?UQPcl;dqJJ`UZqOPG5hVu`+I}cOWpJAK zH9*q-nOEw^aSvZ;t8{y?(A1VrQ!MJyP`gvWR2{2+mhS62NS3ks(UfTkG+Nh|R^cH$ zrz$a1vbKql2J6{iSkJm)`xTJxR@74r+3zQ#Ioqhy!r+aLxNa%4XaY7Ag1-BbT-g&7 zEnA43Sf$`XX|P)icBFFtaJsZTaN4tT^2HymnMH`r%t_cweT6vUr^v%UbLJqZzHngm zs2sD5IrDUk@x9`wk#%l0Dywv5#!4JJ>ZAmIcVbW3Z_JG^d(fLxJ{kPok7R4UGBODcQsLOc6WrI6IZX2P4K`6r#&mGgj_)bsg=2g%V)*85@?fheWZ9TqA_@C%=q>IsK?1&0y() zbtZDCDRUdh$&kjF-i#?}aXST34W(kw_RLm9|D~4qOvZNM*a2%DGl z-q`80D&h(5mEITGPj5B*De+n+y+lCCr$oX9lfI(`d1=M|HxTYuPXq^3Cjw{YAsyGq z6{;^J&H9vhtKTW2hrG4{Un|BU?L#@eL$f&t#%nnUjrKL^wV>ySJ2fB5BRwyg`weN* zGi1+8rIsFmA+|G!(IZ`1uNySXg@p|vta1)*1u=ZXtcoZ_JUuoeGQ96o7=`0A)_%Qm9_1nAmRF=I%O*k5|`w7BsQtIHFw5;pnxfjP)|OZ__t+PushfXaw z0@gEBld0W{4fu)&K5Z_#7o6buLhUUA=WAR>BT;d+Gl^Pm0IUZ@;s=%G3sbk={)XsC z>f`#!b7P7-bx%>t^?>y{YqTaq!c})u6rQHV?=|E_o;aYxjB{jlk}I%iKw6%$@N?dn zt8`F)x<@tp_VBT0Ys`^YGVPXmiwFD0)WP6+NDAamp2BU+-IXE1gvBk5(drh|vjs=Z zSDeSaTZu0}T5)@Wp;}Y)fDnKKJcRmh$dQ_k^fL29Fu>J+!7~R|xZA56hfQWU&a_%j z1bFZif8ZkLvSO;g$E!?3=^CMtXpgjX8kwyBjhj+imypXiOdn7H8tOJnMD}sxyH)1g zbJd&E#V4|cz820Tx0j+5FKN!UMn?*Pd^_yLAF+9?WPb(XE;U7R*`hjKMdaT6I{E9u z*7Z>Pr&|M5FbMGQ9iqX~hG5!qJzsU+=)8SY^TX7zUH1(_4Zbgs=Vi-oA#09B|BxN2 z-H!mX;jeVc^#FC>b0z2^?4)(tY0o&WhMBA ze#*L8zF$#IkEn-_k9jDSUA~(;OBvNsLA{hzuI4yLA||Bg3Nfj0pofEBUd$tA zp@%vsgjaxZbSHPt%q7y zX>&(a{<+HDvoa?F?(c`^>)a3dl2R6Iv(q{y9|!d`P{S_NJz$YNb=lVwaJyEeQGvt9 z0O)h=VflCMkGDuIS8JA4SGQW4q=v~)itlvF?t(~3JlP<`aD9dP&q6~uW>FYc>hDkg zDOdX`>#K*dos5d|uw}T`HB|@7U5KhdYzv1E8cv`ocKuKRVDs;Sh@2<0bH%MWOL#;2 z%b2_^+I2oh{0#|xzjf6(>(-5Pw++DQtTsqerZHN4lbJQmK3?0#_=a(f>blz1>55Sr zavxL}X7z`ZIvT93e0#aj5Lxn67g(poOlc~qZg{5^VztH(#a^2}U#+L#2u|;A;tvO9 zFHLQ2j$oEQU%=+(@Xvt|&_J#kv7f|XBqA5*tzqhjN!jcL+op`xR=3G9qw4{=`N(;+ znQxl1_;fIZV-ccVf7GXCFC_iSQk$~VT~NAy=`i+mWOtWtvuy3b9=5)YQgHW+`1x!j z6x}kQy6@7Rb#v4fcu`kt9RYCD0&hSpRl1>YxIOC3MfBbIv_t!#uRdKJ*qYKCs#@lE z2c?a^Y+0)I^Hfw=na_OrQx-1{&gPEPvJpT7GwVcP$wYeF>sbyFaqRdy=O;GTv$Re8 zhH}H!%!(X4-}!SU_fKN2AiNUQP=tNbo^O@Tg>yc8Ce9wU*(NKV4<{_-1%AbMe6Q4n)U3Um zYNYnv)(H)@v#~vPWp%ytv@5*-wK4(u8~0BuMQ01H-&tQWW8*fgtqMc58X{riu_iw6 zu7H?Gf8y7HkQ@;}y0@4|`zVXPpu6|B-%X@bEYHnVfX)%wJ`}dnrQsH9P@94bSw_-1AqO8E1j(;d>#54uL`*q1;T8<7@v4(y;f{= zPT}VK8zp^PRY@XCX0@}}rxgQl`(B$Ks34~v*m5(tS5h_i=~+zoB2C>2A{WO@ZtE3o zsJ|nlaOi{BVInGiu=4|y^{=9BbGjFUP!U6BO&Bc|rIuKIGvYp64}!9);r;uMvPCVj zb__UN`yp&&u2=6f_&-!RDa(|Yg+F0CPe!37DDL^>Ai*t4Mtkg02|gXY)7_a~AE${x<^vM~mtqn8YV z(avLCuZ}}v&XF3Fh;(Lotv*Y3b#z8K+P|bTYFQ)VHhZDnRf&@U$1g-yrr|B;k1~VQ zrq4StvtIo#>}92Ll|CkJ`Kh#q+q63Rv!Fw)QyNa`d|r<<97jNM+>;PoNVA&`^%fP4 zzmkmfI@jNC`aY5UEQyVwcB98i9ax?EUDa>`8M}fZFuQqvePz5%!eS1Od@^qpFFNX| xV5^i$_U3kvTzuYsKUzRE3w8bf|B=*~JVdvvk=xOayhOg3<42urUmiXm^FK8ku#x}( literal 0 HcmV?d00001 diff --git a/reference-architectures/stackit-kubernetes/buildingblock/main.tf b/reference-architectures/stackit-kubernetes/buildingblock/main.tf new file mode 100644 index 00000000..c0824212 --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/main.tf @@ -0,0 +1,54 @@ +module "cluster" { + source = "github.com/meshcloud/meshstack-hub//modules/stackit/ske/buildingblock?ref=${var.hub.git_ref}" + + stackit_project_id = var.stackit_project_id + stackit_region = var.stackit_region + service_account_email = var.stackit_service_account_email + cluster_name = var.cluster_name + + # The whole DNS design lives in dns.tf. + dns_extension = local.ske_dns_extension +} + +# Registers the cluster as a meshStack platform whose tenants are namespaces, together with the +# dev and prod namespace landing zones the module brings by default. The platform name equals the +# cluster name, so several ordered clusters can coexist in one meshStack instance. +module "platform" { + source = "github.com/meshcloud/meshstack-hub//modules/kubernetes/platform/buildingblock?ref=${var.hub.git_ref}" + + kube_host = module.cluster.kube_host + cluster_ca_certificate = module.cluster.provider_config.cluster_ca_certificate + client_certificate = module.cluster.provider_config.client_certificate + client_key = module.cluster.provider_config.client_key + + owning_workspace_identifier = var.owning_workspace_identifier + location_identifier = var.location_identifier + + platform_name = var.cluster_name + platform_display_name = "Kubernetes namespace on ${var.cluster_name}" + platform_description = "Provides a Kubernetes namespace on the STACKIT Kubernetes Engine cluster ${var.cluster_name}, with a shared HTTPS ingress." +} + +# cert-manager, the HAProxy ingress controller and the Let's Encrypt ClusterIssuer. With a +# delegated DNS subzone the module also issues one wildcard certificate for that subzone, which +# HAProxy serves for every application hostname. +module "ingress" { + count = var.expose == "none" ? 0 : 1 + source = "github.com/meshcloud/meshstack-hub//modules/kubernetes/ingress/buildingblock?ref=${var.hub.git_ref}" + + providers = { + kubernetes = kubernetes + helm = helm + } + + acme_email = var.acme_email + acme_server = var.acme_server + cluster_issuer_name = var.cluster_issuer_name + ingress_class_name = var.ingress_class_name + + # STACKIT reads this annotation on the controller Service and keeps the load balancer off the + # public internet. + haproxy_service_annotations = var.expose == "internal" ? { "lb.stackit.cloud/internal-lb" = "true" } : {} + + dns01 = local.ingress_dns01 +} diff --git a/reference-architectures/stackit-kubernetes/buildingblock/outputs.tf b/reference-architectures/stackit-kubernetes/buildingblock/outputs.tf new file mode 100644 index 00000000..00d4f8a5 --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/outputs.tf @@ -0,0 +1,45 @@ +output "cluster_name" { + description = "Name of the SKE cluster." + value = module.cluster.cluster_name +} + +output "cluster_url" { + description = "Deep link to the STACKIT project that holds the cluster." + value = "https://portal.stackit.cloud/projects/${var.stackit_project_id}" +} + +output "platform_identifier" { + description = "Identifier of the Kubernetes platform registered in meshStack, in the `.` form." + value = module.platform.platform_identifier +} + +output "landing_zones" { + description = "Namespace landing zones created for the platform." + value = join(", ", values(module.platform.landing_zone_identifiers)) +} + +output "apps_domain" { + description = "Delegated DNS subzone of this cluster. Application hostnames live under it and the wildcard certificate covers it. Empty when the architecture runs without DNS." + value = local.dns_enabled ? local.dns_delegated_zone_name : "" +} + +output "ingress_ip" { + description = "External address of the HAProxy load balancer. Empty when `expose` is `none`." + value = var.expose == "none" ? "" : one(module.ingress[*].haproxy_lb_ip) +} + +output "ingress_class_name" { + description = "IngressClass an application puts on its Ingress to be served by this cluster's controller. Empty when `expose` is `none`." + value = var.expose == "none" ? "" : one(module.ingress[*].ingress_class_name) +} + +output "cluster_issuer_name" { + description = "ClusterIssuer an application references from the `cert-manager.io/cluster-issuer` annotation on its Ingress. Empty when `expose` is `none`." + value = var.expose == "none" ? "" : one(module.ingress[*].cluster_issuer_name) +} + +output "kubeconfig" { + description = "Kubeconfig of the cluster. Composed architectures consume it as a static input, for example to install a gateway by Helm." + value = yamlencode(module.cluster.kubeconfig) + sensitive = true +} diff --git a/reference-architectures/stackit-kubernetes/buildingblock/provider.tf b/reference-architectures/stackit-kubernetes/buildingblock/provider.tf new file mode 100644 index 00000000..fd5b1bf8 --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/provider.tf @@ -0,0 +1,18 @@ +# The ingress module is installed only when `expose` is not `none`, so it is called with `count`. +# A module that carries its own provider configuration cannot be called with `count`, which is why +# this root configures the cluster credentials and passes both providers down. +provider "kubernetes" { + host = module.cluster.provider_config.host + cluster_ca_certificate = module.cluster.provider_config.cluster_ca_certificate + client_certificate = module.cluster.provider_config.client_certificate + client_key = module.cluster.provider_config.client_key +} + +provider "helm" { + kubernetes = { + host = module.cluster.provider_config.host + cluster_ca_certificate = module.cluster.provider_config.cluster_ca_certificate + client_certificate = module.cluster.provider_config.client_certificate + client_key = module.cluster.provider_config.client_key + } +} diff --git a/reference-architectures/stackit-kubernetes/buildingblock/variables.tf b/reference-architectures/stackit-kubernetes/buildingblock/variables.tf new file mode 100644 index 00000000..8a72387e --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/variables.tf @@ -0,0 +1,112 @@ +variable "stackit_project_id" { + type = string + nullable = false + description = "STACKIT project UUID of the meshTenant the cluster is created in. meshStack fills this from the STACKIT Project platform tenant." +} + +variable "stackit_region" { + type = string + nullable = false + default = "eu01" + description = "STACKIT region the cluster is created in." +} + +variable "stackit_service_account_email" { + type = string + nullable = true + default = null + description = "Email of the STACKIT service account the SKE module authenticates as through workload identity federation. The account needs `ske.admin` on the organization, because the target project of an order is unknown when the platform team registers the building block." +} + +variable "cluster_name" { + type = string + nullable = false + description = "Name of the SKE cluster. It is also the meshStack platform name and the label of the delegated DNS subzone, so it has to be unique across the landing zone. STACKIT limits SKE cluster names to 11 characters." + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9-]*[a-z0-9])?$", var.cluster_name)) && length(var.cluster_name) <= 11 + error_message = "cluster_name may contain up to 11 lowercase letters, digits and hyphens, and must start and end with a letter or a digit." + } +} + +variable "expose" { + type = string + nullable = false + default = "public" + description = "How the cluster's ingress is reachable. `public` puts the HAProxy ingress controller behind a public load balancer, `internal` keeps the load balancer inside the STACKIT network, and `none` installs no ingress controller at all." + + validation { + condition = contains(["public", "internal", "none"], var.expose) + error_message = "expose must be one of public, internal or none." + } +} + +variable "owning_workspace_identifier" { + type = string + nullable = false + description = "Identifier of the meshStack workspace that owns the Kubernetes platform and its namespace landing zones." +} + +variable "location_identifier" { + type = string + nullable = false + default = "global" + description = "Identifier of the meshStack location the Kubernetes platform is registered in." +} + +variable "acme_email" { + type = string + nullable = false + default = "" + description = "Contact address Let's Encrypt uses for expiry warnings and account recovery. Leave empty to register the ACME account without a contact address." +} + +variable "acme_server" { + type = string + nullable = false + default = "https://acme-v02.api.letsencrypt.org/directory" + description = "ACME directory URL. Point this at the Let's Encrypt staging endpoint while you test, because the production endpoint has strict rate limits." +} + +variable "cluster_issuer_name" { + type = string + nullable = false + default = "letsencrypt-prod" + description = "Name of the ClusterIssuer application teams reference from the `cert-manager.io/cluster-issuer` annotation on their Ingress." +} + +variable "ingress_class_name" { + type = string + nullable = false + default = "haproxy" + description = "Name of the IngressClass the controller serves." +} + +variable "dns_parent_zone_name" { + type = string + nullable = false + default = "" + description = "Parent DNS zone the landing zone owns, for example `likvid.stackit.run`. Every cluster gets a delegated subzone `.` under it. Leave empty to run without DNS, in which case cert-manager issues per-hostname certificates over HTTP-01. See dns.tf." +} + +variable "dns_service_account_key" { + type = string + nullable = false + default = "" + sensitive = true + description = "STACKIT service account key JSON, scoped to the tenant's own STACKIT project, that the cert-manager DNS-01 solver authenticates with. Required for the wildcard certificate. See dns.tf." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { git_ref = "main", bbd_draft = true } + + description = <<-EOT + `git_ref`: meshstack-hub reference used to source the SKE, Kubernetes platform and ingress modules. `const` so it can be interpolated into the module source at init time. + `bbd_draft`: Carried for symmetry with the other hub integrations. This building block registers no building block definition of its own. + EOT +} diff --git a/reference-architectures/stackit-kubernetes/buildingblock/versions.tf b/reference-architectures/stackit-kubernetes/buildingblock/versions.tf new file mode 100644 index 00000000..5c9a9946 --- /dev/null +++ b/reference-architectures/stackit-kubernetes/buildingblock/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.12.0" # const variables require OpenTofu >= 1.12 / Terraform >= 1.15 + + required_providers { + helm = { + source = "hashicorp/helm" + version = ">= 3.0.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.38.0" + } + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.20.0" + } + } +} diff --git a/reference-architectures/stackit-kubernetes/meshstack_integration.tf b/reference-architectures/stackit-kubernetes/meshstack_integration.tf new file mode 100644 index 00000000..37bc4c95 --- /dev/null +++ b/reference-architectures/stackit-kubernetes/meshstack_integration.tf @@ -0,0 +1,403 @@ +variable "stackit_service_account_email" { + type = string + nullable = false + default = "" + description = "Email of the STACKIT service account the building block authenticates as through workload identity federation. It needs `ske.admin` on the organization, because the target project of an order is unknown when the building block definition is registered." +} + +variable "stackit_region" { + type = string + nullable = false + default = "eu01" + description = "STACKIT region the clusters are created in." +} + +variable "stackit_dns_parent_zone_name" { + type = string + nullable = false + default = "" + description = "Parent DNS zone the landing zone owns, for example `likvid.stackit.run`. Each ordered cluster gets a delegated subzone `.` under it. Leave empty to run without DNS, in which case cert-manager issues per-hostname certificates over HTTP-01." +} + +variable "stackit_dns_service_account_key" { + type = string + nullable = false + default = "" + sensitive = true + description = "STACKIT service account key JSON, scoped to the tenant's own STACKIT project, that the cert-manager DNS-01 solver authenticates with. Required for the wildcard certificate." +} + +variable "location_identifier" { + type = string + nullable = false + default = "global" + description = "Identifier of the meshStack location the Kubernetes platforms are registered in." +} + +variable "acme_email" { + type = string + nullable = false + default = "" + description = "Contact address Let's Encrypt uses for expiry warnings and account recovery. Leave empty to register the ACME account without a contact address." +} + +variable "acme_server" { + type = string + nullable = false + default = "https://acme-v02.api.letsencrypt.org/directory" + description = "ACME directory URL. Point this at the Let's Encrypt staging endpoint while you test, because the production endpoint has strict rate limits." +} + +variable "cluster_issuer_name" { + type = string + nullable = false + default = "letsencrypt-prod" + description = "Name of the ClusterIssuer application teams reference from the `cert-manager.io/cluster-issuer` annotation on their Ingress." +} + +variable "ingress_class_name" { + type = string + nullable = false + default = "haproxy" + description = "Name of the IngressClass the ingress controller serves." +} + +variable "meshstack" { + type = object({ + owning_workspace_identifier = string + tags = optional(map(list(string)), {}) + }) + description = "Shared meshStack context. Tags are optional and propagated to building block definition metadata." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { git_ref = "main", bbd_draft = true } + + description = <<-EOT + `git_ref`: Hub release reference. Set to a tag (e.g. 'v1.2.3') or branch or commit sha of the meshstack-hub repo. + `bbd_draft`: If true, the building block definition version is kept in draft mode. + EOT +} + +output "building_block_definition" { + description = "BBD is consumed in building block compositions." + value = { + uuid = meshstack_building_block_definition.this.metadata.uuid + version_ref = var.hub.bbd_draft ? meshstack_building_block_definition.this.version_latest : meshstack_building_block_definition.this.version_latest_release + } +} + +resource "meshstack_building_block_definition" "this" { + metadata = { + owned_by_workspace = var.meshstack.owning_workspace_identifier + tags = var.meshstack.tags + } + + spec = { + display_name = "STACKIT Kubernetes Cluster" + symbol = "https://raw.githubusercontent.com/meshcloud/meshstack-hub/${var.hub.git_ref}/reference-architectures/stackit-kubernetes/buildingblock/logo.png" + description = "Creates an SKE cluster in the tenant's own STACKIT project, installs cert-manager and the HAProxy ingress controller on it, and registers it in meshStack as a Kubernetes platform with namespace landing zones." + support_url = "https://portal.stackit.cloud" + target_type = "TENANT_LEVEL" + run_transparency = true + supported_platforms = [{ name = "STACKIT" }] + + readme = chomp(<<-EOT + This building block creates a **STACKIT Kubernetes Engine cluster** inside your own STACKIT + project and turns it into a platform your teams can order namespaces on. One order gives you + the cluster, an ingress controller with Let's Encrypt certificates, and a Kubernetes platform + in meshStack with a development and a production namespace landing zone. + + An application that runs in one of those namespaces reaches the internet over a hostname in the + cluster's own DNS subzone, and the certificate for that hostname already exists. You do not + request a certificate, and you do not create a DNS record. + + ## 🎯 When to use it + + Use this building block when you: + - need a Kubernetes cluster of your own on STACKIT rather than a namespace on a shared cluster. + - want your teams to order namespaces from the meshStack catalog instead of asking you for them. + - want every application to get an HTTPS hostname without a certificate request. + + ## πŸ’‘ Usage examples + + **Example 1: A team platform** + A team orders the cluster in its STACKIT project and names it `team-a`. meshStack registers the + cluster as a platform with a `team-a-dev` and a `team-a-prod` landing zone, and the team's + projects get namespaces on it with the quotas the landing zone grants. + + **Example 2: A cluster that stays inside the network** + A team runs an internal application and sets **Ingress Exposure** to `internal`. The load + balancer in front of the ingress controller receives a private address, so the application is + reachable from inside the STACKIT network only, while certificates are still issued and renewed + automatically. + + ## 🌐 Hostnames and certificates + + The cluster receives a delegated DNS subzone named after it, for example + `team-a.likvid.stackit.run`. The SKE managed ExternalDNS extension writes the records, and + cert-manager holds one wildcard certificate for the whole subzone. Adding an Ingress with a + hostname under that subzone is all an application needs. + + ## πŸ“Š Shared Responsibility + + | Responsibility | Platform Team | Application Team | + |---|:---:|:---:| + | Provide the STACKIT identity the building block runs as | βœ… | ❌ | + | Own the parent DNS zone and delegate a subzone per cluster | βœ… | ❌ | + | Register and maintain this building block definition | βœ… | ❌ | + | Choose the cluster name and the ingress exposure | ❌ | βœ… | + | Order namespaces on the resulting Kubernetes platform | ❌ | βœ… | + | Deploy, expose and operate the applications in those namespaces | ❌ | βœ… | + EOT + ) + } + + version_spec = { + draft = var.hub.bbd_draft + deletion_mode = "DELETE" + + # Ephemeral API key permissions for the meshStack platform and landing zones the composed + # `modules/kubernetes/platform` module registers. + permissions = [ + "LANDINGZONE_LIST", + "LANDINGZONE_SAVE", + "LANDINGZONE_DELETE", + "PLATFORMINSTANCE_LIST", + "PLATFORMINSTANCE_SAVE", + "PLATFORMINSTANCE_DELETE" + ] + + implementation = { + terraform = { + terraform_version = "1.12.0" + repository_url = "https://github.com/meshcloud/meshstack-hub.git" + repository_path = "reference-architectures/stackit-kubernetes/buildingblock" + ref_name = var.hub.git_ref + async = false + use_mesh_http_backend_fallback = true + } + } + + inputs = { + # ── Tenant context and STACKIT authentication ── + + stackit_project_id = { + display_name = "STACKIT Project ID" + description = "STACKIT project of the meshTenant the cluster is created in." + type = "STRING" + assignment_type = "PLATFORM_TENANT_ID" + } + + stackit_service_account_email = { + display_name = "Service Account Email" + description = "Email of the STACKIT service account for WIF-based authentication." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_service_account_email) + } + + STACKIT_USE_OIDC = { + display_name = "STACKIT Use OIDC" + description = "Enables OIDC-based WIF for the STACKIT provider." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("1") + } + + STACKIT_FEDERATED_TOKEN_FILE = { + display_name = "STACKIT Federated Token File" + description = "Path to the WIF token file injected by meshStack." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("/var/run/secrets/workload-identity/azure/token") + } + + stackit_region = { + display_name = "STACKIT Region" + description = "STACKIT region the cluster is created in." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_region) + } + + hub = { + display_name = "Hub" + description = "JSON object with `git_ref`, the meshstack-hub reference used to source the SKE, Kubernetes platform and ingress modules." + type = "CODE" + assignment_type = "STATIC" + argument = jsonencode(jsonencode(var.hub)) + } + + # ── What the application team decides ── + + cluster_name = { + display_name = "Cluster Name" + description = "Name of the cluster. It also names the meshStack platform and the cluster's DNS subzone. STACKIT limits SKE cluster names to 11 characters." + type = "STRING" + assignment_type = "USER_INPUT" + value_validation_regex = "^[a-z0-9]([a-z0-9-]{0,9}[a-z0-9])?$" + validation_regex_error_message = "Cluster name may contain up to 11 lowercase letters, digits and hyphens, and must start and end with a letter or a digit." + } + + expose = { + display_name = "Ingress Exposure" + description = "`public` puts the ingress controller behind a public load balancer, `internal` keeps the load balancer inside the STACKIT network, and `none` installs no ingress controller." + type = "STRING" + assignment_type = "USER_INPUT" + updateable_by_consumer = true + default_value = jsonencode("public") + value_validation_regex = "^(public|internal|none)$" + validation_regex_error_message = "Ingress exposure must be public, internal or none." + } + + # ── meshStack platform registration ── + + owning_workspace_identifier = { + display_name = "Workspace Identifier" + description = "Workspace that owns the Kubernetes platform and its namespace landing zones." + type = "STRING" + assignment_type = "WORKSPACE_IDENTIFIER" + } + + location_identifier = { + display_name = "Location Identifier" + description = "meshStack location the Kubernetes platform is registered in." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.location_identifier) + } + + # ── Ingress and certificates, set once by the platform team ── + + acme_email = { + display_name = "ACME Contact Email" + description = "Contact address Let's Encrypt uses for expiry warnings and account recovery." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.acme_email) + } + + acme_server = { + display_name = "ACME Directory URL" + description = "ACME directory URL. Point this at the Let's Encrypt staging endpoint while you test." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.acme_server) + } + + cluster_issuer_name = { + display_name = "Cluster Issuer Name" + description = "Name of the ClusterIssuer applications reference from their Ingress." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.cluster_issuer_name) + } + + ingress_class_name = { + display_name = "Ingress Class Name" + description = "Name of the IngressClass the controller serves." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.ingress_class_name) + } + + # ── DNS ── + + dns_parent_zone_name = { + display_name = "DNS Parent Zone" + description = "Parent zone the landing zone owns. Each cluster gets a delegated subzone under it. Leave empty to run without DNS." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_dns_parent_zone_name) + } + + dns_service_account_key = { + display_name = "DNS Service Account Key" + description = "STACKIT service account key JSON the cert-manager DNS-01 solver authenticates with, scoped to the tenant's own project." + type = "STRING" + assignment_type = "STATIC" + sensitive = { + argument = { + secret_value = var.stackit_dns_service_account_key + } + } + } + } + + outputs = { + cluster_name = { + display_name = "Cluster Name" + type = "STRING" + assignment_type = "NONE" + } + + cluster_url = { + display_name = "Open STACKIT Project" + type = "STRING" + assignment_type = "RESOURCE_URL" + } + + platform_identifier = { + display_name = "Kubernetes Platform" + type = "STRING" + assignment_type = "NONE" + } + + landing_zones = { + display_name = "Namespace Landing Zones" + type = "STRING" + assignment_type = "NONE" + } + + apps_domain = { + display_name = "Application Domain" + type = "STRING" + assignment_type = "NONE" + } + + ingress_ip = { + display_name = "Ingress Address" + type = "STRING" + assignment_type = "NONE" + } + + ingress_class_name = { + display_name = "Ingress Class" + type = "STRING" + assignment_type = "NONE" + } + + cluster_issuer_name = { + display_name = "Cluster Issuer" + type = "STRING" + assignment_type = "NONE" + } + + kubeconfig = { + display_name = "Kubeconfig" + type = "STRING" + assignment_type = "NONE" + is_sensitive = true + } + } + } +} + +terraform { + required_version = ">= 1.12.0" + + required_providers { + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.24.0" + } + } +} From 4d47b15a3116579818e1b1849861313fc0509e26 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:24:03 +0200 Subject: [PATCH 20/60] feat: add STACKIT PostgreSQL Flex building block Reference architectures must not declare stackit_postgresflex_instance inline, so this adds a hub module that owns the resource. The module is sourceable as a plain Terraform module by a composition and orderable as a building block through meshstack_integration.tf. The buildingblock creates the instance, one database and one owning user, and returns host, port, database name, username, the generated password and a connection string that LiteLLM and Langfuse take as DATABASE_URL. STACKIT encodes CPU, RAM and node count in one flavor ID, so the module takes flavor_cpu, flavor_ram and replicas separately and resolves the matching flavor through the stackit_postgresflex_flavors data source. The deprecated flavor block stays unused. The ACL defaults to the STACKIT service ranges 193.148.160.0/19, 45.129.40.0/21 and 45.135.244.0/22. An SKE cluster reaches the instance through a NAT router, so the team adds the cluster egress IP as a /32 entry; allow_stackit_public_ip_ranges pulls the live list instead. 0.0.0.0/0 is rejected. Default PostgreSQL version is 17 and anything below 16 is rejected, because version 14 reaches end of life on 12 November 2026. The backplane grants postgres-flex.admin at organization scope over a WIF service account, the same shape modules/stackit/network/backplane uses for a TENANT_LEVEL building block. Designed against stackitcloud/stackit v0.110.0. Co-Authored-By: Claude Opus 5 (1M context) --- .../stackit/postgresflex/backplane/README.md | 80 ++++ .../stackit/postgresflex/backplane/main.tf | 38 ++ .../stackit/postgresflex/backplane/outputs.tf | 4 + .../postgresflex/backplane/variables.tf | 27 ++ .../postgresflex/backplane/versions.tf | 10 + .../postgresflex/buildingblock/README.md | 124 ++++++ .../buildingblock/SUMMARY.md.tftpl | 22 + .../postgresflex/buildingblock/logo.png | Bin 0 -> 1878 bytes .../postgresflex/buildingblock/main.tf | 77 ++++ .../postgresflex/buildingblock/outputs.tf | 52 +++ .../postgresflex/buildingblock/provider.tf | 8 + .../postgresflex/buildingblock/variables.tf | 203 ++++++++++ .../postgresflex/buildingblock/versions.tf | 12 + .../postgresflex/meshstack_integration.tf | 381 ++++++++++++++++++ 14 files changed, 1038 insertions(+) create mode 100644 modules/stackit/postgresflex/backplane/README.md create mode 100644 modules/stackit/postgresflex/backplane/main.tf create mode 100644 modules/stackit/postgresflex/backplane/outputs.tf create mode 100644 modules/stackit/postgresflex/backplane/variables.tf create mode 100644 modules/stackit/postgresflex/backplane/versions.tf create mode 100644 modules/stackit/postgresflex/buildingblock/README.md create mode 100644 modules/stackit/postgresflex/buildingblock/SUMMARY.md.tftpl create mode 100644 modules/stackit/postgresflex/buildingblock/logo.png create mode 100644 modules/stackit/postgresflex/buildingblock/main.tf create mode 100644 modules/stackit/postgresflex/buildingblock/outputs.tf create mode 100644 modules/stackit/postgresflex/buildingblock/provider.tf create mode 100644 modules/stackit/postgresflex/buildingblock/variables.tf create mode 100644 modules/stackit/postgresflex/buildingblock/versions.tf create mode 100644 modules/stackit/postgresflex/meshstack_integration.tf diff --git a/modules/stackit/postgresflex/backplane/README.md b/modules/stackit/postgresflex/backplane/README.md new file mode 100644 index 00000000..9f4988dd --- /dev/null +++ b/modules/stackit/postgresflex/backplane/README.md @@ -0,0 +1,80 @@ +# STACKIT PostgreSQL Flex – Backplane + +This module sets up the shared backplane configuration for the STACKIT PostgreSQL Flex building +block. It creates a dedicated service account with a Workload Identity Federation (WIF) identity +provider and the permission required to create PostgreSQL Flex instances in any project under a +given organization: + +- **`postgres-flex.admin`** β€” allows creating and managing PostgreSQL Flex instances, databases and users. + +Authentication uses WIF (OIDC token exchange) β€” no long-lived service account key is created or stored. + +## Why the role is assigned at organization scope + +The building block is `TENANT_LEVEL`, so it creates the instance in the tenant's own STACKIT project. +The platform team deploys this backplane once, long before it knows which projects future instances +will land in, so the role cannot be scoped to a single project. STACKIT offers no predefined role +narrower than `postgres-flex.admin` for creating instances. A custom role is the least-privilege +alternative once the exact permission set is stable. + +This mirrors `modules/stackit/network/backplane`, which grants `iaas.network.admin` at organization +scope for the same reason. + +## Prerequisites + +- A STACKIT project where the service account will be created. +- A STACKIT service account with permissions to manage service accounts and organization-level role assignments. +- The STACKIT organization ID under which target projects live. +- meshStack WIF issuer and subject from `data.meshstack_integrations.integrations`. + +## Usage + +```hcl +module "postgresflex_backplane" { + source = "./backplane" + + project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + organization_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + + workload_identity_federation = { + issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer + subjects = [""] + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.11.0 | +| [stackit](#requirement\_stackit) | >= 0.110.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_authorization_organization_role_assignment.postgres_flex_admin](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_organization_role_assignment) | resource | +| [stackit_service_account.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account) | resource | +| [stackit_service_account_federated_identity_provider.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account_federated_identity_provider) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [organization\_id](#input\_organization\_id) | STACKIT organization ID where the service account will be granted permissions to manage PostgreSQL Flex instances. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | STACKIT project ID where the service account will be created. | `string` | n/a | yes | +| [service\_account\_name](#input\_service\_account\_name) | Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project. | `string` | `"mesh-postgresflex"` | no | +| [workload\_identity\_federation](#input\_workload\_identity\_federation) | WIF issuer URL and subject list for the meshStack building block identity provider. | 

object({
    issuer   = string
    subjects = list(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [service\_account\_email](#output\_service\_account\_email) | Email of the STACKIT service account used by the buildingblock provider via WIF. | + diff --git a/modules/stackit/postgresflex/backplane/main.tf b/modules/stackit/postgresflex/backplane/main.tf new file mode 100644 index 00000000..82a15079 --- /dev/null +++ b/modules/stackit/postgresflex/backplane/main.tf @@ -0,0 +1,38 @@ +resource "stackit_service_account" "building_block" { + project_id = var.project_id + name = var.service_account_name +} + +resource "stackit_service_account_federated_identity_provider" "building_block" { + for_each = { for i, s in var.workload_identity_federation.subjects : tostring(i) => s } + + project_id = var.project_id + service_account_email = stackit_service_account.building_block.email + name = "meshstack-${each.key}" + issuer = var.workload_identity_federation.issuer + + assertions = [ + { + item = "aud" + operator = "equals" + value = "api://AzureADTokenExchange" + }, + { + item = "sub" + operator = "equals" + value = each.value + } + ] +} + +# postgres-flex.admin at org scope allows creating instances, users and databases in any tenant +# project under the organization. Required because this is a TENANT_LEVEL building block: the +# backplane is deployed once, before the target project of any future instance is known, so +# permissions can't be scoped to a single project ahead of time. STACKIT offers no narrower +# predefined role than postgres-flex.admin for creating instances; a custom role would be the +# least-privilege alternative once the exact permission set is stable. +resource "stackit_authorization_organization_role_assignment" "postgres_flex_admin" { + resource_id = var.organization_id + role = "postgres-flex.admin" + subject = stackit_service_account.building_block.email +} diff --git a/modules/stackit/postgresflex/backplane/outputs.tf b/modules/stackit/postgresflex/backplane/outputs.tf new file mode 100644 index 00000000..3fde57dd --- /dev/null +++ b/modules/stackit/postgresflex/backplane/outputs.tf @@ -0,0 +1,4 @@ +output "service_account_email" { + value = stackit_service_account.building_block.email + description = "Email of the STACKIT service account used by the buildingblock provider via WIF." +} diff --git a/modules/stackit/postgresflex/backplane/variables.tf b/modules/stackit/postgresflex/backplane/variables.tf new file mode 100644 index 00000000..ff3d097a --- /dev/null +++ b/modules/stackit/postgresflex/backplane/variables.tf @@ -0,0 +1,27 @@ +variable "project_id" { + type = string + nullable = false + description = "STACKIT project ID where the service account will be created." +} + +variable "organization_id" { + type = string + nullable = false + description = "STACKIT organization ID where the service account will be granted permissions to manage PostgreSQL Flex instances." +} + +variable "workload_identity_federation" { + type = object({ + issuer = string + subjects = list(string) + }) + nullable = false + description = "WIF issuer URL and subject list for the meshStack building block identity provider." +} + +variable "service_account_name" { + type = string + default = "mesh-postgresflex" + nullable = false + description = "Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project." +} diff --git a/modules/stackit/postgresflex/backplane/versions.tf b/modules/stackit/postgresflex/backplane/versions.tf new file mode 100644 index 00000000..5c2057d3 --- /dev/null +++ b/modules/stackit/postgresflex/backplane/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.11.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.110.0" + } + } +} diff --git a/modules/stackit/postgresflex/buildingblock/README.md b/modules/stackit/postgresflex/buildingblock/README.md new file mode 100644 index 00000000..fbcab6fd --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/README.md @@ -0,0 +1,124 @@ +--- +name: STACKIT PostgreSQL Flex +supportedPlatforms: + - stackit +description: Provisions a managed STACKIT PostgreSQL Flex instance with a database and a database user. +--- + +# STACKIT PostgreSQL Flex Building Block + +This building block module creates a managed PostgreSQL Flex instance in an existing STACKIT +project, plus one database and one database user that owns it. It returns the host, the port, the +database name, the user, the generated password and a ready-to-use connection string. + +The module is used in two ways. A reference architecture sources it directly as a Terraform module, +which keeps STACKIT resources out of the architecture itself. An application team orders it as a +building block through meshStack, wired up by the `meshstack_integration.tf` at the module root. + +## Choosing the instance shape + +STACKIT encodes CPU, RAM and node count in a single flavor ID. This module takes `flavor_cpu`, +`flavor_ram` and `replicas` as separate inputs and resolves the matching flavor through the +`stackit_postgresflex_flavors` data source, because `flavor_id` is the only field the provider still +supports. `replicas` is 1 for a single node or 3 for a replicated instance; STACKIT accepts no other +value. When no flavor matches, the module fails during plan and lists the flavors the project offers. + +## Reaching the instance β€” the ACL + +A PostgreSQL Flex instance rejects every connection whose source address is not covered by its ACL. +Getting this wrong is the usual reason a workload times out against a fresh instance. + +The `acl` input defaults to `193.148.160.0/19`, `45.129.40.0/21` and `45.135.244.0/22`. These are the +STACKIT service ranges that STACKIT documents as the predefined ACL of its data services, so the +default lets other STACKIT services reach the instance. The default does **not** cover arbitrary +clients on the internet, and it does not cover a client in your office network. + +An SKE cluster does not connect from a private address. All egress traffic of a cluster leaves +through a single router that applies NAT, so the instance sees the cluster's public egress IPv4 +address. STACKIT picks that address from its pool when the cluster is created and it stays fixed for +the cluster's lifetime, but STACKIT does not document that it falls inside the ranges above. Read the +egress IP from the cluster's overview page in the STACKIT Portal and add it as a `/32` entry: + +```hcl +acl = ["193.148.160.0/19", "45.129.40.0/21", "45.135.244.0/22", "203.0.113.17/32"] +``` + +Set `allow_stackit_public_ip_ranges = true` to add every public range STACKIT publishes, read live +from the `stackit_public_ip_ranges` data source. That list is the one STACKIT keeps current, at the +cost of a much wider allowlist. + +The module rejects `0.0.0.0/0`, because STACKIT documents that entry as one to avoid. + +`network_access_scope` selects `PUBLIC` or `SNA`. `SNA` is in private preview and STACKIT rejects the +request when the project is not enabled for it, so the input is unset by default. + +## PostgreSQL version + +STACKIT PostgreSQL Flex **version 14 reaches end of life on 12 November 2026**, and version 13 is +already past its end of life. The module therefore defaults to version 17 and refuses anything below +16. Version 16 is supported until November 2028. + +## Backups + +STACKIT takes one backup per day at the time given by `backup_schedule`, a cron expression whose +minute and hour must be numeric. `retention_days` decides how long STACKIT keeps them and must be +between 32 and 90. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.11.0 | +| [stackit](#requirement\_stackit) | >= 0.110.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_postgresflex_database.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/postgresflex_database) | resource | +| [stackit_postgresflex_instance.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/postgresflex_instance) | resource | +| [stackit_postgresflex_user.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/postgresflex_user) | resource | +| [stackit_postgresflex_flavors.available](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/data-sources/postgresflex_flavors) | data source | +| [stackit_public_ip_ranges.stackit](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/data-sources/public_ip_ranges) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [acl](#input\_acl) | Source IPv4 CIDR ranges allowed to open a connection to the instance. The default is the set of
STACKIT service ranges that STACKIT documents as the predefined ACL for its data services, so
other STACKIT services can reach the instance.

An SKE cluster leaves its network through a router that applies NAT, so the instance sees the
cluster's public egress IP and not a private address. Add that egress IP as a /32 entry when the
cluster's range is not already covered here. Never add 0.0.0.0/0 β€” STACKIT documents that as
something to avoid, because it opens the instance to every address on the internet. | `list(string)` | 
[
  "193.148.160.0/19",
  "45.129.40.0/21",
  "45.135.244.0/22"
]
| no | +| [allow\_stackit\_public\_ip\_ranges](#input\_allow\_stackit\_public\_ip\_ranges) | Add every public IP range STACKIT publishes to the ACL, on top of `acl`. This reads the `stackit_public_ip_ranges` data source, which is the machine-readable list STACKIT keeps current. Turn it on when the hardcoded default in `acl` goes stale, at the cost of a much wider allowlist. | `bool` | `false` | no | +| [backup\_schedule](#input\_backup\_schedule) | Cron expression that decides when STACKIT takes the daily backup. Minute and hour must be numeric, for example '0 2 * * *' for 02:00 UTC. | `string` | `"0 2 * * *"` | no | +| [database\_name](#input\_database\_name) | Name of the database created on the instance. | `string` | `"app"` | no | +| [database\_user\_roles](#input\_database\_user\_roles) | Roles granted to the database user. STACKIT supports `login` and `createdb`. | `list(string)` | 
[
  "login",
  "createdb"
]
| no | +| [database\_username](#input\_database\_username) | Name of the database user that owns the database. STACKIT generates the password and this module returns it as a sensitive output. | `string` | `"app"` | no | +| [flavor\_cpu](#input\_flavor\_cpu) | Number of vCPUs of the instance flavor. Together with `flavor_ram` and `replicas` this selects a STACKIT flavor. STACKIT offers 2/4, 4/8, 16/32, 2/16, 4/32, 16/128 and 8/16 as CPU/RAM pairs. | `number` | `2` | no | +| [flavor\_ram](#input\_flavor\_ram) | Memory of the instance flavor in GiB. Must form a valid pair with `flavor_cpu`. | `number` | `4` | no | +| [instance\_name](#input\_instance\_name) | Name of the PostgreSQL Flex instance. | `string` | n/a | yes | +| [network\_access\_scope](#input\_network\_access\_scope) | Network access scope of the instance, either `PUBLIC` or `SNA`. Leave unset to get the STACKIT default. `SNA` is in private preview and STACKIT rejects the request when the project is not enabled for it. | `string` | `null` | no | +| [postgres\_version](#input\_postgres\_version) | PostgreSQL major version. Version 14 reaches end of life on 12 November 2026, so pick 16 or newer. | `string` | `"17"` | no | +| [project\_id](#input\_project\_id) | STACKIT project ID the PostgreSQL Flex instance is created in. | `string` | n/a | yes | +| [replicas](#input\_replicas) | Number of nodes. 1 creates a single-node instance, 3 creates a replicated instance. STACKIT supports no other value. | `number` | `1` | no | +| [retention\_days](#input\_retention\_days) | Number of days STACKIT keeps backups. STACKIT accepts 32 to 90. | `number` | `32` | no | +| [service\_account\_email](#input\_service\_account\_email) | Email of the STACKIT service account the provider authenticates as via workload identity federation. Leave unset when the caller supplies its own provider configuration. | `string` | `null` | no | +| [stackit\_region](#input\_stackit\_region) | STACKIT region the instance is created in. Ignored when the caller supplies its own provider configuration. | `string` | `"eu01"` | no | +| [storage\_class](#input\_storage\_class) | Storage performance class. STACKIT offers premium-perf2-stackit through premium-perf12-stackit, with higher numbers giving more IOPS and throughput. | `string` | `"premium-perf2-stackit"` | no | +| [storage\_size](#input\_storage\_size) | Storage size of the instance in GB. | `number` | `20` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [connection\_string](#output\_connection\_string) | Ready-to-use libpq connection string including the password. Applications such as LiteLLM and Langfuse take this as their DATABASE\_URL. | +| [database\_name](#output\_database\_name) | Name of the database created on the instance. | +| [host](#output\_host) | DNS name of the instance's write endpoint. | +| [instance\_id](#output\_instance\_id) | UUID of the PostgreSQL Flex instance. | +| [password](#output\_password) | Password of the database user. STACKIT generates it and shows it only once. | +| [port](#output\_port) | TCP port of the instance's write endpoint. | +| [summary](#output\_summary) | Summary with instance details and database credentials. | +| [username](#output\_username) | Name of the database user that owns the database. | + diff --git a/modules/stackit/postgresflex/buildingblock/SUMMARY.md.tftpl b/modules/stackit/postgresflex/buildingblock/SUMMARY.md.tftpl new file mode 100644 index 00000000..c1d1c805 --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/SUMMARY.md.tftpl @@ -0,0 +1,22 @@ +# PostgreSQL Flex: **${instance_name}** + +## Details + +| Property | Value | +|----------|-------| +| **Instance ID** | `${instance_id}` | +| **PostgreSQL Version** | `${version}` | +| **Host** | `${host}` | +| **Port** | `${port}` | +| **Database** | `${database_name}` | +| **Username** | `${username}` | +| **Password** | `${password}` | +| **Allowed Source Ranges** | `${acl}` | + +## Connect + +```sh +psql "postgresql://${username}@${host}:${port}/${database_name}?sslmode=require" +``` + +Your client must reach the instance from one of the allowed source ranges listed above. diff --git a/modules/stackit/postgresflex/buildingblock/logo.png b/modules/stackit/postgresflex/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..225391782a085772b79e9ce3d491e62e933a940b GIT binary patch literal 1878 zcmZ8i2~?6<6#mQHCCjnK(l(cza%3*YfuRsEa0DL$4O$-xtp}b4MMWVo zfeMjAp^y-y03^Mv1$zUw z4poaVh9R*ztJC4?ulWj;;bZ0`Y)R7wF>gasHDB$$xQupA>x&glpWun7U!X z6yQ-nUiT1VI|15|w@CB~(tsaHH7}U30rKe!DYTvXWx~+sH{JTfGnX5D{Cm<)ZwzVm z26H|C-IKMKgpD=n^)GKu?72mac~WN-#!cL^CfN1oUA{|~ESk$PnQdWa!`#fD<*_~1 z;aOXD_4(g#yq0BUJiPGWeNNtJ$?>Kl(bf4s1{zPR>_|N_eSv2ZAv!MJ*)eL0U2-HZ z!g7_w>Li>k+$@!KgpE?hYsxj z<92SQebBlJK|$~0{N5)kPC(G4$#|bN8+WwzswtEWYb^Dj&vlsKow3)Xk$S+OZX$Jx zsZoKzX!<0dT5q@EXZ-%^k!So)>1bTu@R?<7QDe`XZExtwk9^tfx!YIzY!4ZFbstJw zImi!q?+fSOXT={=QI0D-HN1O^EwtwD*ac^5^IR$%OX5a8aJ9}BlAB)e96#2GbycBh zK+h@er#Q#-Y9Dx{q^i*W_3%9Vp-%!ePFTy>GEgYN_sBFu0a?8`5ksGGm$Vx`=%v{YowgOH%u7%C?e2AtBelE91ma+(vh#i~)A{CG zvN*-n7%P#g?+9ygkDrHRIEdKyyZ1Y*Qi|2H>sfADs*X%WB&J+zYoU#H$70Xc!UK&F z2J$50F;zliYql+K$wpXOv_tr`eqr5`DsFJ3Wy8RqXD;s)QtDls@0@M8V^tRaywr`n zO6KG^0VliqxYaQ>@5QOX>1A%6rK{2OHvKZ`FUs=hql1h?uE{o51{E5(^*oW9>bl~G)8J69EY7}oU3@HN8@=x^hCf3R zYPZM?mRCoYGUL6o<$4Hv_9jM)2~QkTQw;Nz9P6PZ-ZHl3A`C3gEoR31Quz92G8I=9 zr@XCbc`IN7pu`$swO;rQl&z0CO58kO5{5$>+RK>new09*Y0y3u*O8-C+1|CZ^$)3Z zdHOQKRGszL@*cu+ufB$mh{`?&VuEmqva1%ni&s}yaSI{<&rF@ zpIo7N+tl5oBJ-zpBz?8oD*xc;%@IF)aP8{_muwlkU&ax$DqSc(?0z{%Y*tqumQV?Q zF2GXYH0i>ad*@v!f!-^k4iHmgX$A$ABX?wCC+B5A&2q zf~v9-C8muV?wkZNhQ{r7U+b*Dgy7xHHAi#bMAaFH5#Pl1ls`I1qPza&yut}6k*-ZB zWtDvvHK~iJb`_=SB4q*zW{YW4x61YRYodHB&JjStC3PR5-ub`Y^Tq0-j{$_MVw#8m zBR&l7j}mptdjT|#V9^uR5o#oQ@f=t~|Nuh`oom fvBSwdOZPyP@64?#ZB$msKSI1O!RLl|#NK}ZQo#tq literal 0 HcmV?d00001 diff --git a/modules/stackit/postgresflex/buildingblock/main.tf b/modules/stackit/postgresflex/buildingblock/main.tf new file mode 100644 index 00000000..d20a4847 --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/main.tf @@ -0,0 +1,77 @@ +data "stackit_postgresflex_flavors" "available" { + project_id = var.project_id + region = var.stackit_region +} + +data "stackit_public_ip_ranges" "stackit" { + count = var.allow_stackit_public_ip_ranges ? 1 : 0 +} + +locals { + # The splat yields an empty list when the data source is switched off, so no conditional is needed. + acl = distinct(concat(var.acl, flatten(data.stackit_public_ip_ranges.stackit[*].cidr_list))) + + # STACKIT encodes the node count in the flavor, so a single flavor ID carries CPU, RAM and + # "Single" or "Replica" at once. The module takes the three as separate inputs and resolves the + # matching flavor here, because `flavor_id` is the only non-deprecated way to set the shape. + node_type = var.replicas == 1 ? "Single" : "Replica" + + matching_flavors = [ + for flavor in data.stackit_postgresflex_flavors.available.flavors : flavor + if flavor.cpu == var.flavor_cpu && flavor.memory == var.flavor_ram && flavor.node_type == local.node_type + ] + + flavor_id = length(local.matching_flavors) == 1 ? local.matching_flavors[0].id : null + + available_flavors = join(", ", [ + for flavor in data.stackit_postgresflex_flavors.available.flavors : + "${flavor.cpu} CPU / ${flavor.memory} GiB / ${flavor.node_type}" + ]) +} + +resource "stackit_postgresflex_instance" "this" { + project_id = var.project_id + region = var.stackit_region + name = var.instance_name + + flavor_id = local.flavor_id + version = var.postgres_version + + storage = { + class = var.storage_class + size = var.storage_size + } + + backup_schedule = var.backup_schedule + retention_days = var.retention_days + + # An instance is unreachable until the source range of the client is listed here. An SKE cluster + # leaves its network behind a NAT router, so the instance sees the cluster's public egress IP. + network = { + acl = local.acl + access_scope = var.network_access_scope + } + + lifecycle { + precondition { + condition = local.flavor_id != null + error_message = "No STACKIT flavor matches ${var.flavor_cpu} CPU, ${var.flavor_ram} GiB RAM and ${var.replicas} replica(s). Available flavors: ${local.available_flavors}." + } + } +} + +resource "stackit_postgresflex_user" "this" { + project_id = var.project_id + region = var.stackit_region + instance_id = stackit_postgresflex_instance.this.instance_id + username = var.database_username + roles = var.database_user_roles +} + +resource "stackit_postgresflex_database" "this" { + project_id = var.project_id + region = var.stackit_region + instance_id = stackit_postgresflex_instance.this.instance_id + name = var.database_name + owner = stackit_postgresflex_user.this.username +} diff --git a/modules/stackit/postgresflex/buildingblock/outputs.tf b/modules/stackit/postgresflex/buildingblock/outputs.tf new file mode 100644 index 00000000..1cd6e5eb --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/outputs.tf @@ -0,0 +1,52 @@ +output "instance_id" { + value = stackit_postgresflex_instance.this.instance_id + description = "UUID of the PostgreSQL Flex instance." +} + +output "host" { + value = stackit_postgresflex_instance.this.connection_info.write.host + description = "DNS name of the instance's write endpoint." +} + +output "port" { + value = stackit_postgresflex_instance.this.connection_info.write.port + description = "TCP port of the instance's write endpoint." +} + +output "database_name" { + value = stackit_postgresflex_database.this.name + description = "Name of the database created on the instance." +} + +output "username" { + value = stackit_postgresflex_user.this.username + description = "Name of the database user that owns the database." +} + +output "password" { + value = stackit_postgresflex_user.this.password + description = "Password of the database user. STACKIT generates it and shows it only once." + sensitive = true +} + +output "connection_string" { + value = "postgresql://${stackit_postgresflex_user.this.username}:${urlencode(stackit_postgresflex_user.this.password)}@${stackit_postgresflex_instance.this.connection_info.write.host}:${stackit_postgresflex_instance.this.connection_info.write.port}/${stackit_postgresflex_database.this.name}?sslmode=require" + description = "Ready-to-use libpq connection string including the password. Applications such as LiteLLM and Langfuse take this as their DATABASE_URL." + sensitive = true +} + +output "summary" { + description = "Summary with instance details and database credentials." + sensitive = true + value = templatefile("${path.module}/SUMMARY.md.tftpl", { + instance_name = stackit_postgresflex_instance.this.name + instance_id = stackit_postgresflex_instance.this.instance_id + host = stackit_postgresflex_instance.this.connection_info.write.host + port = stackit_postgresflex_instance.this.connection_info.write.port + database_name = stackit_postgresflex_database.this.name + username = stackit_postgresflex_user.this.username + password = stackit_postgresflex_user.this.password + version = stackit_postgresflex_instance.this.version + acl = join(", ", local.acl) + }) +} diff --git a/modules/stackit/postgresflex/buildingblock/provider.tf b/modules/stackit/postgresflex/buildingblock/provider.tf new file mode 100644 index 00000000..b8dfb483 --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/provider.tf @@ -0,0 +1,8 @@ +# Callers that drive this module from their own root configuration usually replace this file with a +# generated `provider.tf`. In that case `service_account_email` and `stackit_region` stay unset and +# the generated block carries the credentials instead. +provider "stackit" { + default_region = var.stackit_region + service_account_email = var.service_account_email + use_oidc = true +} diff --git a/modules/stackit/postgresflex/buildingblock/variables.tf b/modules/stackit/postgresflex/buildingblock/variables.tf new file mode 100644 index 00000000..a51766af --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/variables.tf @@ -0,0 +1,203 @@ +# ── Backplane inputs (static, set once per building block definition) ────────── + +variable "project_id" { + type = string + nullable = false + description = "STACKIT project ID the PostgreSQL Flex instance is created in." +} + +variable "service_account_email" { + type = string + nullable = true + default = null + description = "Email of the STACKIT service account the provider authenticates as via workload identity federation. Leave unset when the caller supplies its own provider configuration." +} + +variable "stackit_region" { + type = string + nullable = false + default = "eu01" + description = "STACKIT region the instance is created in. Ignored when the caller supplies its own provider configuration." +} + +# ── Instance shape ───────────────────────────────────────────────────────────── + +variable "instance_name" { + type = string + nullable = false + description = "Name of the PostgreSQL Flex instance." + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9-]*[a-z0-9])?$", var.instance_name)) + error_message = "The instance name may contain lowercase letters, digits and hyphens, and must start and end with a letter or a digit." + } +} + +variable "flavor_cpu" { + type = number + nullable = false + default = 2 + description = "Number of vCPUs of the instance flavor. Together with `flavor_ram` and `replicas` this selects a STACKIT flavor. STACKIT offers 2/4, 4/8, 16/32, 2/16, 4/32, 16/128 and 8/16 as CPU/RAM pairs." +} + +variable "flavor_ram" { + type = number + nullable = false + default = 4 + description = "Memory of the instance flavor in GiB. Must form a valid pair with `flavor_cpu`." +} + +variable "replicas" { + type = number + nullable = false + default = 1 + description = "Number of nodes. 1 creates a single-node instance, 3 creates a replicated instance. STACKIT supports no other value." + + validation { + condition = contains([1, 3], var.replicas) + error_message = "replicas must be either 1 (single node) or 3 (replicated)." + } +} + +variable "storage_class" { + type = string + nullable = false + default = "premium-perf2-stackit" + description = "Storage performance class. STACKIT offers premium-perf2-stackit through premium-perf12-stackit, with higher numbers giving more IOPS and throughput." +} + +variable "storage_size" { + type = number + nullable = false + default = 20 + description = "Storage size of the instance in GB." + + validation { + condition = var.storage_size >= 5 + error_message = "storage_size must be at least 5 GB." + } +} + +variable "postgres_version" { + type = string + nullable = false + default = "17" + description = "PostgreSQL major version. Version 14 reaches end of life on 12 November 2026, so pick 16 or newer." + + validation { + condition = can(regex("^(1[6-9]|[2-9][0-9])$", var.postgres_version)) + error_message = "postgres_version must be 16 or newer. Versions 13, 14 and 15 are end of life or close to it." + } +} + +# ── Backup ───────────────────────────────────────────────────────────────────── + +variable "backup_schedule" { + type = string + nullable = false + default = "0 2 * * *" + description = "Cron expression that decides when STACKIT takes the daily backup. Minute and hour must be numeric, for example '0 2 * * *' for 02:00 UTC." +} + +variable "retention_days" { + type = number + nullable = false + default = 32 + description = "Number of days STACKIT keeps backups. STACKIT accepts 32 to 90." + + validation { + condition = var.retention_days >= 32 && var.retention_days <= 90 + error_message = "retention_days must be between 32 and 90." + } +} + +# ── Network access ───────────────────────────────────────────────────────────── + +variable "acl" { + type = list(string) + nullable = false + default = ["193.148.160.0/19", "45.129.40.0/21", "45.135.244.0/22"] + + description = <<-EOT + Source IPv4 CIDR ranges allowed to open a connection to the instance. The default is the set of + STACKIT service ranges that STACKIT documents as the predefined ACL for its data services, so + other STACKIT services can reach the instance. + + An SKE cluster leaves its network through a router that applies NAT, so the instance sees the + cluster's public egress IP and not a private address. Add that egress IP as a /32 entry when the + cluster's range is not already covered here. Never add 0.0.0.0/0 β€” STACKIT documents that as + something to avoid, because it opens the instance to every address on the internet. + EOT + + validation { + condition = length(var.acl) > 0 + error_message = "acl must contain at least one CIDR range, otherwise nothing can reach the instance." + } + + validation { + condition = alltrue([for cidr in var.acl : can(cidrnetmask(cidr))]) + error_message = "Every entry of acl must be a valid IPv4 CIDR range, for example 45.129.40.0/21." + } + + validation { + condition = !contains(var.acl, "0.0.0.0/0") + error_message = "acl must not contain 0.0.0.0/0. STACKIT documents that entry as one to avoid, because it lets every address on the internet reach the instance. List the ranges your clients actually use." + } +} + +variable "allow_stackit_public_ip_ranges" { + type = bool + nullable = false + default = false + description = "Add every public IP range STACKIT publishes to the ACL, on top of `acl`. This reads the `stackit_public_ip_ranges` data source, which is the machine-readable list STACKIT keeps current. Turn it on when the hardcoded default in `acl` goes stale, at the cost of a much wider allowlist." +} + +variable "network_access_scope" { + type = string + nullable = true + default = null + description = "Network access scope of the instance, either `PUBLIC` or `SNA`. Leave unset to get the STACKIT default. `SNA` is in private preview and STACKIT rejects the request when the project is not enabled for it." + + validation { + condition = var.network_access_scope == null ? true : contains(["PUBLIC", "SNA"], var.network_access_scope) + error_message = "network_access_scope must be either PUBLIC or SNA." + } +} + +# ── Database and user ────────────────────────────────────────────────────────── + +variable "database_name" { + type = string + nullable = false + default = "app" + description = "Name of the database created on the instance." + + validation { + condition = can(regex("^[a-z_][a-z0-9_]*$", var.database_name)) + error_message = "The database name may contain lowercase letters, digits and underscores, and must not start with a digit." + } +} + +variable "database_username" { + type = string + nullable = false + default = "app" + description = "Name of the database user that owns the database. STACKIT generates the password and this module returns it as a sensitive output." + + validation { + condition = can(regex("^[a-z_][a-z0-9_]*$", var.database_username)) + error_message = "The username may contain lowercase letters, digits and underscores, and must not start with a digit." + } +} + +variable "database_user_roles" { + type = list(string) + nullable = false + default = ["login", "createdb"] + description = "Roles granted to the database user. STACKIT supports `login` and `createdb`." + + validation { + condition = alltrue([for role in var.database_user_roles : contains(["login", "createdb"], role)]) + error_message = "database_user_roles may only contain 'login' and 'createdb'." + } +} diff --git a/modules/stackit/postgresflex/buildingblock/versions.tf b/modules/stackit/postgresflex/buildingblock/versions.tf new file mode 100644 index 00000000..888a306e --- /dev/null +++ b/modules/stackit/postgresflex/buildingblock/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1.11.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + # 0.110.0 is the version this module was written against. Earlier versions do not carry the + # v3 PostgreSQL Flex schema with `flavor_id`, `network.acl` and `retention_days`. + version = ">= 0.110.0" + } + } +} diff --git a/modules/stackit/postgresflex/meshstack_integration.tf b/modules/stackit/postgresflex/meshstack_integration.tf new file mode 100644 index 00000000..cee078bd --- /dev/null +++ b/modules/stackit/postgresflex/meshstack_integration.tf @@ -0,0 +1,381 @@ +variable "stackit_organization_id" { + type = string + description = "STACKIT organization ID under which target projects live." +} + +variable "stackit_project_id" { + type = string + description = "STACKIT project ID where the backplane service account will be created." +} + +variable "stackit_service_account_name" { + type = string + default = null + description = "Name of the backplane service account. Defaults to 'mesh-postgresflex'. Override when deploying multiple backplane instances in the same STACKIT project." +} + +variable "stackit_region" { + type = string + default = "eu01" + description = "STACKIT region the PostgreSQL Flex instances are created in." +} + +variable "stackit_postgresflex_acl" { + type = list(string) + default = ["193.148.160.0/19", "45.129.40.0/21", "45.135.244.0/22"] + + description = <<-EOT + Source IPv4 CIDR ranges offered as the default ACL of an ordered instance. The default is the set + of STACKIT service ranges that STACKIT documents as the predefined ACL for its data services. An + SKE cluster reaches the instance through a NAT router, so the application team adds the cluster's + public egress IP as a /32 entry when it is not already covered. + EOT +} + +variable "meshstack" { + type = object({ + owning_workspace_identifier = string + tags = optional(map(list(string)), {}) + }) + description = "Shared meshStack context." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { + git_ref = "main" + bbd_draft = true + } + description = <<-EOT + `git_ref`: Hub release reference. Set to a tag (e.g. 'v1.2.3') or branch or commit sha of meshcloud/meshstack-hub repo. + `bbd_draft`: If true, allows changing the building block definition for upgrading dependent building blocks. + EOT +} + +output "building_block_definition" { + description = "BBD is consumed in building block compositions." + value = { + uuid = meshstack_building_block_definition.this.metadata.uuid + version_ref = var.hub.bbd_draft ? meshstack_building_block_definition.this.version_latest : meshstack_building_block_definition.this.version_latest_release + } +} + +data "meshstack_integrations" "integrations" {} + +module "backplane" { + source = "github.com/meshcloud/meshstack-hub//modules/stackit/postgresflex/backplane?ref=${var.hub.git_ref}" + + project_id = var.stackit_project_id + organization_id = var.stackit_organization_id + service_account_name = coalesce(var.stackit_service_account_name, "mesh-postgresflex") + + workload_identity_federation = { + issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer + subjects = [ + "${trimsuffix(data.meshstack_integrations.integrations.workload_identity_federation.replicator.subject, ":replicator")}:workspace.${var.meshstack.owning_workspace_identifier}.buildingblockdefinition.${meshstack_building_block_definition.this.metadata.uuid}" + ] + } +} + +resource "meshstack_building_block_definition" "this" { + metadata = { + owned_by_workspace = var.meshstack.owning_workspace_identifier + tags = var.meshstack.tags + } + + spec = { + display_name = "STACKIT PostgreSQL Flex" + symbol = "https://raw.githubusercontent.com/meshcloud/meshstack-hub/${var.hub.git_ref}/modules/stackit/postgresflex/buildingblock/logo.png" + description = "Provisions a managed STACKIT PostgreSQL Flex instance with a database and a database user." + support_url = "https://portal.stackit.cloud" + target_type = "TENANT_LEVEL" + run_transparency = true + supported_platforms = [{ name = "STACKIT" }] + readme = chomp(<<-EOT + This building block provisions a **managed PostgreSQL Flex instance on STACKIT** inside your + STACKIT project, together with one database and one database user, so your application gets a + relational database without anyone running a Postgres server. + + ## 🎯 When to use it + + Use this building block when you: + - Need a managed PostgreSQL database for an application running in your STACKIT project. + - Want STACKIT to take the daily backups and to keep the engine patched. + - Prefer a single connection string over assembling host, port, user and password yourself. + + ## πŸ’‘ Usage examples + + **Example 1: Database for a workload on SKE** + A team runs an API on an SKE cluster and orders this building block for its persistent data. + The team puts the cluster's public egress IP into the ACL and feeds the connection string into + the deployment as an environment variable. + + **Example 2: Database for an AI gateway** + An AI platform runs LiteLLM and Langfuse, and both need their own PostgreSQL. The team orders + this building block twice, once per component, and hands each the connection string as its + `DATABASE_URL`. + + ## πŸ”’ Reaching the instance + + A PostgreSQL Flex instance only accepts connections from the source ranges listed in its ACL. + The default covers the STACKIT service ranges. An SKE cluster leaves its network through a + router that applies NAT, so the instance sees the cluster's public egress IP rather than a + private address β€” add that address as a `/32` entry when your cluster is not already covered. + You find the egress IP in the STACKIT Portal under the cluster's overview. Do not add + `0.0.0.0/0`, it would open the instance to every address on the internet. + + ## πŸ—“οΈ Version support + + PostgreSQL 14 reaches end of life on **12 November 2026**, and 13 is already past it. Order + version 16 or newer. + + ## πŸ“Š Shared Responsibility + + | Responsibility | Platform Team | Application Team | + |---|:---:|:---:| + | Provide the backplane identity used to create the instance | βœ… | ❌ | + | Run the managed PostgreSQL service, its patches and its backups | βœ… | ❌ | + | Offer a sensible default ACL of STACKIT service ranges | βœ… | ❌ | + | Choose instance size, storage, PostgreSQL version and backup schedule | ❌ | βœ… | + | Add the client source ranges the application connects from | ❌ | βœ… | + | Database schema, migrations and query performance | ❌ | βœ… | + | Keep the returned credentials secret and rotate them when needed | ❌ | βœ… | + EOT + ) + } + + version_spec = { + draft = var.hub.bbd_draft + deletion_mode = "DELETE" + + implementation = { + terraform = { + terraform_version = "1.11.0" + repository_url = "https://github.com/meshcloud/meshstack-hub.git" + repository_path = "modules/stackit/postgresflex/buildingblock" + ref_name = var.hub.git_ref + async = false + use_mesh_http_backend_fallback = true + } + } + + inputs = { + project_id = { + display_name = "STACKIT Project ID" + description = "STACKIT project ID of the existing project the instance will be created in." + type = "STRING" + assignment_type = "PLATFORM_TENANT_ID" + } + + service_account_email = { + display_name = "Service Account Email" + description = "Email of the STACKIT service account for WIF-based authentication." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(module.backplane.service_account_email) + } + + stackit_region = { + display_name = "STACKIT Region" + description = "STACKIT region the instance is created in." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_region) + } + + STACKIT_USE_OIDC = { + display_name = "STACKIT Use OIDC" + description = "Enables OIDC-based WIF for the STACKIT provider." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("1") + } + + STACKIT_FEDERATED_TOKEN_FILE = { + display_name = "STACKIT Federated Token File" + description = "Path to the WIF token file injected by meshStack." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("/var/run/secrets/workload-identity/azure/token") + } + + instance_name = { + display_name = "Instance Name" + description = "Name of the PostgreSQL Flex instance." + type = "STRING" + assignment_type = "USER_INPUT" + value_validation_regex = "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" + validation_regex_error_message = "The instance name may contain lowercase letters, digits and hyphens, and must start and end with a letter or a digit." + } + + flavor_cpu = { + display_name = "vCPUs" + description = "Number of vCPUs. Valid CPU/RAM pairs are 2/4, 4/8, 16/32, 2/16, 4/32, 16/128 and 8/16." + type = "INTEGER" + assignment_type = "USER_INPUT" + default_value = jsonencode(2) + } + + flavor_ram = { + display_name = "Memory (GiB)" + description = "Memory in GiB. Must form a valid pair with the number of vCPUs." + type = "INTEGER" + assignment_type = "USER_INPUT" + default_value = jsonencode(4) + } + + replicas = { + display_name = "Replicas" + description = "Number of nodes. 1 creates a single-node instance, 3 creates a replicated instance." + type = "INTEGER" + assignment_type = "USER_INPUT" + default_value = jsonencode(1) + value_validation_regex = "^(1|3)$" + validation_regex_error_message = "Replicas must be either 1 or 3." + } + + storage_class = { + display_name = "Storage Class" + description = "Storage performance class, from premium-perf2-stackit up to premium-perf12-stackit." + type = "STRING" + assignment_type = "USER_INPUT" + default_value = jsonencode("premium-perf2-stackit") + value_validation_regex = "^premium-perf(2|4|6|8|10|12)-stackit$" + validation_regex_error_message = "Storage class must be one of premium-perf2-stackit, premium-perf4-stackit, premium-perf6-stackit, premium-perf8-stackit, premium-perf10-stackit, premium-perf12-stackit." + } + + storage_size = { + display_name = "Storage Size (GB)" + description = "Storage size of the instance in GB." + type = "INTEGER" + assignment_type = "USER_INPUT" + default_value = jsonencode(20) + } + + postgres_version = { + display_name = "PostgreSQL Version" + description = "PostgreSQL major version. Version 14 reaches end of life on 12 November 2026, so pick 16 or newer." + type = "STRING" + assignment_type = "USER_INPUT" + default_value = jsonencode("17") + value_validation_regex = "^(1[6-9]|[2-9][0-9])$" + validation_regex_error_message = "The PostgreSQL version must be 16 or newer." + } + + backup_schedule = { + display_name = "Backup Schedule" + description = "Cron expression that decides when STACKIT takes the daily backup. Minute and hour must be numeric, for example '0 2 * * *' for 02:00 UTC." + type = "STRING" + assignment_type = "USER_INPUT" + default_value = jsonencode("0 2 * * *") + } + + retention_days = { + display_name = "Backup Retention (days)" + description = "Number of days STACKIT keeps backups. STACKIT accepts 32 to 90." + type = "INTEGER" + assignment_type = "USER_INPUT" + default_value = jsonencode(32) + } + + acl = { + display_name = "Allowed Source Ranges" + description = "JSON list of IPv4 CIDR ranges allowed to connect. Keep the STACKIT service ranges and add your SKE cluster's public egress IP as a /32 entry. Do not add 0.0.0.0/0." + type = "CODE" + assignment_type = "USER_INPUT" + default_value = jsonencode(jsonencode(var.stackit_postgresflex_acl)) + } + + database_name = { + display_name = "Database Name" + description = "Name of the database created on the instance." + type = "STRING" + assignment_type = "USER_INPUT" + default_value = jsonencode("app") + value_validation_regex = "^[a-z_][a-z0-9_]*$" + validation_regex_error_message = "The database name may contain lowercase letters, digits and underscores, and must not start with a digit." + } + + database_username = { + display_name = "Database Username" + description = "Name of the database user that owns the database." + type = "STRING" + assignment_type = "USER_INPUT" + default_value = jsonencode("app") + value_validation_regex = "^[a-z_][a-z0-9_]*$" + validation_regex_error_message = "The username may contain lowercase letters, digits and underscores, and must not start with a digit." + } + } + + outputs = { + host = { + display_name = "Host" + type = "STRING" + assignment_type = "NONE" + } + + port = { + display_name = "Port" + type = "INTEGER" + assignment_type = "NONE" + } + + database_name = { + display_name = "Database Name" + type = "STRING" + assignment_type = "NONE" + } + + username = { + display_name = "Username" + type = "STRING" + assignment_type = "NONE" + } + + password = { + display_name = "Password" + type = "STRING" + assignment_type = "NONE" + } + + connection_string = { + display_name = "Connection String" + type = "STRING" + assignment_type = "NONE" + } + + instance_id = { + display_name = "Instance ID" + type = "STRING" + assignment_type = "NONE" + } + + summary = { + display_name = "Summary" + type = "STRING" + assignment_type = "SUMMARY" + } + } + } +} + +terraform { + required_version = ">= 1.12.0" + + required_providers { + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.21.0" + } + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.110.0" + } + } +} From 1e2b26efc75d4bef78808e7a927f8a5375160ac7 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:24:20 +0200 Subject: [PATCH 21/60] feat(stackit/storage-bucket): add endpoint output and multipart permissions Langfuse configures an S3 client from a bare endpoint URL plus a bucket name, which neither bucket_url_path_style nor bucket_url_virtual_hosted_style provides. The new endpoint output returns the host on its own. A local now holds the endpoint so the path-style URL and the summary derive from one place; the region stays hardcoded to eu01, the same value provider.tf uses. The bucket policy also gains s3:AbortMultipartUpload, s3:ListMultipartUploadParts and s3:ListBucketMultipartUploads. Creating, uploading and completing parts is covered by s3:PutObject, but cancelling an upload and listing leftover parts are separate actions. The AWS SDK upload helpers call AbortMultipartUpload when an upload fails part-way, so without these the abort returns 403 and the orphaned parts stay in the bucket and keep costing storage. Co-Authored-By: Claude Opus 5 (1M context) --- .../storage-bucket/buildingblock/README.md | 1 + .../buildingblock/SUMMARY.md.tftpl | 1 + .../stackit/storage-bucket/buildingblock/main.tf | 7 +++++++ .../storage-bucket/buildingblock/outputs.tf | 16 ++++++++++++++-- .../storage-bucket/meshstack_integration.tf | 6 ++++++ 5 files changed, 29 insertions(+), 2 deletions(-) diff --git a/modules/stackit/storage-bucket/buildingblock/README.md b/modules/stackit/storage-bucket/buildingblock/README.md index d312f8a9..69121d09 100644 --- a/modules/stackit/storage-bucket/buildingblock/README.md +++ b/modules/stackit/storage-bucket/buildingblock/README.md @@ -49,6 +49,7 @@ No modules. | [bucket\_name](#output\_bucket\_name) | Name of the created Object Storage bucket. | | [bucket\_url\_path\_style](#output\_bucket\_url\_path\_style) | Path-style URL of the bucket. | | [bucket\_url\_virtual\_hosted\_style](#output\_bucket\_url\_virtual\_hosted\_style) | Virtual-hosted-style URL of the bucket. | +| [endpoint](#output\_endpoint) | Base URL of the STACKIT Object Storage S3 endpoint, without the bucket name. Use this for S3 clients that take an endpoint and a bucket separately. | | [s3\_access\_key](#output\_s3\_access\_key) | S3-compatible access key for the bucket. | | [s3\_secret\_access\_key](#output\_s3\_secret\_access\_key) | S3-compatible secret access key for the bucket. | | [summary](#output\_summary) | Summary with bucket details and access credentials. | diff --git a/modules/stackit/storage-bucket/buildingblock/SUMMARY.md.tftpl b/modules/stackit/storage-bucket/buildingblock/SUMMARY.md.tftpl index 5b9a06c5..f7918f0c 100644 --- a/modules/stackit/storage-bucket/buildingblock/SUMMARY.md.tftpl +++ b/modules/stackit/storage-bucket/buildingblock/SUMMARY.md.tftpl @@ -5,6 +5,7 @@ | Property | Value | |----------|-------| | **Bucket Name** | `${bucket_name}` | +| **Endpoint** | ${endpoint} | | **Path-Style URL** | ${bucket_url_path_style} | | **Virtual-Hosted URL** | ${bucket_url_virtual_hosted} | | **Access Key** | `${access_key}` | diff --git a/modules/stackit/storage-bucket/buildingblock/main.tf b/modules/stackit/storage-bucket/buildingblock/main.tf index f97699bf..30b6bf73 100644 --- a/modules/stackit/storage-bucket/buildingblock/main.tf +++ b/modules/stackit/storage-bucket/buildingblock/main.tf @@ -50,6 +50,13 @@ resource "aws_s3_bucket_policy" "this" { "s3:PutObject", "s3:DeleteObject", "s3:ListBucket", + # Multipart uploads: creating, uploading and completing parts is covered by s3:PutObject, + # but cancelling an upload and finding leftover parts are separate actions. Clients such as + # the AWS SDK upload helpers call AbortMultipartUpload when an upload fails part-way, and + # without these the abort returns 403 and the orphaned parts stay in the bucket. + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucketMultipartUploads", ] Resource = [ "urn:sgws:s3:::${aws_s3_bucket.this.bucket}", diff --git a/modules/stackit/storage-bucket/buildingblock/outputs.tf b/modules/stackit/storage-bucket/buildingblock/outputs.tf index f9612868..772e3cd7 100644 --- a/modules/stackit/storage-bucket/buildingblock/outputs.tf +++ b/modules/stackit/storage-bucket/buildingblock/outputs.tf @@ -1,10 +1,21 @@ +locals { + # STACKIT serves Object Storage from one endpoint per region. The region is fixed to eu01 here, + # the same value `provider.tf` configures for the S3 provider. + endpoint = "https://object.storage.eu01.onstackit.cloud" +} + output "bucket_name" { value = aws_s3_bucket.this.bucket description = "Name of the created Object Storage bucket." } +output "endpoint" { + value = local.endpoint + description = "Base URL of the STACKIT Object Storage S3 endpoint, without the bucket name. Use this for S3 clients that take an endpoint and a bucket separately." +} + output "bucket_url_path_style" { - value = "https://object.storage.eu01.onstackit.cloud/${aws_s3_bucket.this.bucket}" + value = "${local.endpoint}/${aws_s3_bucket.this.bucket}" description = "Path-style URL of the bucket." } @@ -29,7 +40,8 @@ output "summary" { sensitive = true value = templatefile("${path.module}/SUMMARY.md.tftpl", { bucket_name = aws_s3_bucket.this.bucket - bucket_url_path_style = "https://object.storage.eu01.onstackit.cloud/${aws_s3_bucket.this.bucket}" + endpoint = local.endpoint + bucket_url_path_style = "${local.endpoint}/${aws_s3_bucket.this.bucket}" bucket_url_virtual_hosted = "https://${aws_s3_bucket.this.bucket}.object.storage.eu01.onstackit.cloud" access_key = stackit_objectstorage_credential.this.access_key secret_access_key = stackit_objectstorage_credential.this.secret_access_key diff --git a/modules/stackit/storage-bucket/meshstack_integration.tf b/modules/stackit/storage-bucket/meshstack_integration.tf index bf04b12c..2776768c 100644 --- a/modules/stackit/storage-bucket/meshstack_integration.tf +++ b/modules/stackit/storage-bucket/meshstack_integration.tf @@ -213,6 +213,12 @@ resource "meshstack_building_block_definition" "this" { assignment_type = "NONE" } + endpoint = { + display_name = "S3 Endpoint" + type = "STRING" + assignment_type = "NONE" + } + s3_access_key = { display_name = "S3 Access Key" type = "STRING" From de2f47c2ab49c6daaf928b97d422b83df0e4ce29 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:31:52 +0200 Subject: [PATCH 22/60] docs: add the ai platform directory The website generator reads a README.md with front matter and a logo from every directory under modules/, so the ai modules need both before they can appear in the catalog. The logo is original line art in the style of modules/ske/logo.svg, because AI is a capability grouping and has no vendor mark of its own. The category is devops, which is the closest of the five values the website accepts. Adding an ai category would mean changing the website, which is out of scope for this pull request. Co-Authored-By: Claude Opus 5 (1M context) --- modules/ai/README.md | 10 ++++++++++ modules/ai/logo.svg | 1 + 2 files changed, 11 insertions(+) create mode 100644 modules/ai/README.md create mode 100644 modules/ai/logo.svg diff --git a/modules/ai/README.md b/modules/ai/README.md new file mode 100644 index 00000000..80ca4e53 --- /dev/null +++ b/modules/ai/README.md @@ -0,0 +1,10 @@ +--- +name: AI +description: meshStack integration with AI platforms puts a gateway in front of your model providers, so application teams call one OpenAI-compatible endpoint with a virtual key while the platform team keeps the provider credentials, the budgets and the spend records. +category: devops +benefits: + - Model Gateway + - Virtual Key Issuing + - Budget and Spend Tracking +official: true +--- diff --git a/modules/ai/logo.svg b/modules/ai/logo.svg new file mode 100644 index 00000000..40a4e9e7 --- /dev/null +++ b/modules/ai/logo.svg @@ -0,0 +1 @@ + From 8b60e422b14e9ddfda562070e22c878ef44413ac Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:32:11 +0200 Subject: [PATCH 23/60] feat(ai): install the LiteLLM gateway with the litellm building block The platform team installs the gateway into a Kubernetes namespace and registers the OpenAI-compatible endpoints it fronts. The ai-platform reference architecture sources the module, so it follows the sourced-not-ordered pattern: a buildingblock tier with an APP_TEAM_README.md, no meshstack_integration.tf and no backplane. The chart litellm-helm is published to GHCR as an OCI artifact only and is pinned to 1.96.2, the tag on the registry. Chart.yaml on the repository's main branch carries a different version, because the release pipeline overwrites the field while publishing. Postgres is mandatory, because virtual keys, teams, budgets and spend tracking all live there. The connection arrives as inputs, so the module works with any Postgres, and the chart's bundled Bitnami subchart stays off: those images no longer receive updates. The Prisma migration Job runs as a Helm pre-install hook so the pods never start against a database without the schema. Redis is optional and off by default. It is the coordination store for cross-pod rate limits, spend tracking and the pod lock manager, so a single replica needs none and several replicas with enforced budgets need it. Two traps in the model registration are encoded and validated: the openai/ prefix on the model name selects the OpenAI-compatible driver, and an api_base without the /v1 suffix answers Not Found on every call. Co-Authored-By: Claude Opus 5 (1M context) --- .../litellm/buildingblock/APP_TEAM_README.md | 62 ++++++ modules/ai/litellm/buildingblock/README.md | 201 ++++++++++++++++++ modules/ai/litellm/buildingblock/logo.png | Bin 0 -> 5905 bytes modules/ai/litellm/buildingblock/main.tf | 185 ++++++++++++++++ modules/ai/litellm/buildingblock/outputs.tf | 30 +++ modules/ai/litellm/buildingblock/provider.tf | 15 ++ modules/ai/litellm/buildingblock/variables.tf | 183 ++++++++++++++++ modules/ai/litellm/buildingblock/versions.tf | 16 ++ 8 files changed, 692 insertions(+) create mode 100644 modules/ai/litellm/buildingblock/APP_TEAM_README.md create mode 100644 modules/ai/litellm/buildingblock/README.md create mode 100644 modules/ai/litellm/buildingblock/logo.png create mode 100644 modules/ai/litellm/buildingblock/main.tf create mode 100644 modules/ai/litellm/buildingblock/outputs.tf create mode 100644 modules/ai/litellm/buildingblock/provider.tf create mode 100644 modules/ai/litellm/buildingblock/variables.tf create mode 100644 modules/ai/litellm/buildingblock/versions.tf diff --git a/modules/ai/litellm/buildingblock/APP_TEAM_README.md b/modules/ai/litellm/buildingblock/APP_TEAM_README.md new file mode 100644 index 00000000..9673b2c6 --- /dev/null +++ b/modules/ai/litellm/buildingblock/APP_TEAM_README.md @@ -0,0 +1,62 @@ +Your application reaches the large language models the platform team offers through one OpenAI-compatible endpoint. You get a virtual key and a model alias, and you point any OpenAI client at the gateway. The credentials of the model provider stay with the platform team, and your spend and rate limits are tracked against your own key. + +## 🎯 When to use it + +Use this building block when you: +- want to call a language model from your application without asking a provider for your own account and credential +- need to switch between models, or between providers, without changing your application code +- have to keep a budget per team or per application and see what has been spent +- want the same endpoint in every environment, so a change of model stays a configuration change + +## πŸ’‘ Usage examples + +**Example 1: Add a summarization feature** +Your service summarizes support tickets. You point the OpenAI SDK at the gateway URL, set your virtual key as the API key and ask for the model alias the platform team published. Nothing else in your code changes when the platform team moves that alias to another model. + +**Example 2: Keep an experiment inside a budget** +Your team tries a retrieval feature and does not want to spend more than the budget it was given. The gateway counts every call against your virtual key and refuses further requests once the budget is used up, so an experiment cannot run away with cost. + +## πŸ”§ How to use it + +The gateway speaks the OpenAI API, so every OpenAI client works. Set the base URL to the gateway, including the `/v1` suffix, and use your virtual key as the API key. + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://litellm.litellm.svc.cluster.local:4000/v1", + api_key="sk-your-virtual-key", +) + +response = client.chat.completions.create( + model="chat-large", + messages=[{"role": "user", "content": "Summarize this ticket."}], +) +``` + +The same call with curl: + +```bash +curl http://litellm.litellm.svc.cluster.local:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-your-virtual-key" \ + -H "Content-Type: application/json" \ + -d '{"model": "chat-large", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +Two things go wrong often enough to name them: + +- **Keep the `/v1` in the base URL.** Without it every call answers "Not Found". +- **Use the alias, not the name of the model at the provider.** The gateway resolves the alias to a model and an endpoint, and the alias is the only name it accepts. + +## πŸ“Š Shared Responsibility + +| Responsibility | Platform Team | Application Team | +|---|:---:|:---:| +| Run the gateway and its database | βœ… | ❌ | +| Hold the credentials of the model providers | βœ… | ❌ | +| Decide which models are available and under which alias | βœ… | ❌ | +| Issue virtual keys and set budgets and rate limits | βœ… | ❌ | +| Keep the virtual key secret and rotate it when it leaks | ❌ | βœ… | +| Pick a model alias that fits the task and its cost | ❌ | βœ… | +| Handle rate limit and budget errors in the application | ❌ | βœ… | +| Decide what data the application sends to a model | ❌ | βœ… | diff --git a/modules/ai/litellm/buildingblock/README.md b/modules/ai/litellm/buildingblock/README.md new file mode 100644 index 00000000..04278ee8 --- /dev/null +++ b/modules/ai/litellm/buildingblock/README.md @@ -0,0 +1,201 @@ +--- +name: LiteLLM AI Gateway +supportedPlatforms: + - kubernetes +description: Installs the LiteLLM gateway into a Kubernetes namespace and registers OpenAI-compatible model endpoints behind one API, with virtual keys, teams, budgets and spend tracking backed by Postgres. +# The cluster credentials, the database connection and the upstream model credentials all arrive +# as inputs, so there is nothing to set up on the cloud side before this module runs. +requiresBackplane: false +--- + +# LiteLLM AI Gateway Building Block + +The platform team installs the LiteLLM gateway into a Kubernetes namespace with this module. The gateway puts one OpenAI-compatible API in front of the model endpoints the platform team registers, and it issues virtual keys with their own budgets and rate limits, so an application team never sees the credential of an upstream provider. + +This documentation is intended as a reference for cloud foundation or platform engineers using this module. + +## Sourced, not ordered + +There is no `meshstack_integration.tf` and no `backplane/`. The `ai-platform` reference architecture sources `buildingblock/` from its own building block, and a foundation can source it from a Terragrunt unit. Application teams do not order this module; they order a virtual key from a separate building block that talks to the gateway this module installs. + +## The chart + +| | | +|---|---| +| Chart | `litellm-helm` | +| Reference | `oci://ghcr.io/berriai/litellm-helm` | +| Pinned version | `1.96.2` (`var.chart_version`) | +| Source | [`helm/litellm-helm/`](https://github.com/BerriAI/litellm/tree/main/helm/litellm-helm) in `BerriAI/litellm` | +| Prerequisites | Kubernetes 1.21+, Helm 3.8.0+ | + +Three details about this chart cost time when you meet them for the first time: + +- **The chart is published to GHCR as an OCI artifact only.** There is no classic Helm repository to add. The helm provider takes the registry and the path prefix as `repository` and the chart name on its own as `chart`, which is why `main.tf` reads `repository = "oci://ghcr.io/berriai"` and `chart = "litellm-helm"`. +- **`Chart.yaml` on the repository's main branch lies about the version.** It reads a much lower number, because the release pipeline overwrites the field while publishing. Pick the version from the [GHCR tag list](https://github.com/BerriAI/litellm/pkgs/container/litellm-helm) instead. +- **A second, newer chart lives at `helm/litellm/`.** That one splits the proxy into microservices. This module uses the monolithic `litellm-helm` chart. The old `deploy/charts/` path from older documentation no longer exists. + +## Postgres is required + +Virtual keys, teams, budgets and spend tracking all live in Postgres. Without a database the gateway is a stateless proxy and none of those endpoints work, which is the whole reason to run it here. The module therefore takes the connection as mandatory inputs: `postgres_host`, `postgres_port`, `postgres_database`, `postgres_username` and `postgres_password`. Any Postgres works. On STACKIT a PostgreSQL Flex instance fits; in another foundation an operator-managed cluster in the same Kubernetes cluster fits just as well. + +The chart bundles a Bitnami postgresql subchart under `db.deployStandalone`, and that switch defaults to **true**. This module sets it to `false` and sets `db.useExisting = true` instead. The chart's own README recommends the same: those images no longer receive updates and the subchart is pinned to `bitnamilegacy/postgresql`. + +Four points to keep in mind: + +- **The database has to exist before the first apply.** The migration Job creates the tables inside it, not the database itself. +- **The user needs rights to create and alter tables**, because the migration Job runs the schema migrations under it. +- **The password has to be safe in a URL.** The chart builds the connection URL from `$(DATABASE_USERNAME)` and `$(DATABASE_PASSWORD)`, which Kubernetes substitutes verbatim and does not URL-encode, so a password containing `:`, `@`, `/` or `?` breaks the URL. +- **`postgres_ssl_mode` defaults to `require`.** A managed Postgres terminates TLS and accepts this. A server without TLS needs `prefer` or `disable`. + +The chart's default connection URL carries no port, so the module writes its own `db.url` with `postgres_port` and the sslmode parameter in it. The credentials stay in a Kubernetes Secret and never appear in the pod spec, because the URL keeps the `$(…)` references that Kubernetes resolves from the environment. + +### The Prisma migration Job + +The chart runs the schema migrations in a Prisma Job (`migrationJob.enabled`, default true). Its default annotations address ArgoCD, which means nothing to a Terraform-driven install: the Job would be applied together with the Deployment, and the pods would restart until the schema exists, because the proxy itself runs with `DISABLE_SCHEMA_UPDATE=true` whenever the Job is enabled. + +This module therefore turns the Helm hook on and the ArgoCD annotations off. The Job runs as a `pre-install,pre-upgrade` hook, Helm waits for it to finish, and only then creates the Deployment. Two consequences follow: + +- A failing migration fails the whole `helm_release`, which is what you want β€” the alternative is a pod crash loop that Terraform reports as a timeout. +- The Job needs its share of `helm_timeout`. The default of 600 seconds covers a migration and a rollout together. + +## Redis + +`redis.enabled` defaults to false in the chart and the module keeps the bundled Redis subchart off, for the same reason as the bundled Postgres. Redis is the coordination store of the gateway: cross-pod rate limits, spend tracking and the pod lock manager. + +| Deployment | Redis | +|---|---| +| `replica_count = 1` | Not needed. One pod counts everything in its own memory and the counts are correct. | +| `replica_count > 1` | Required as soon as you enforce budgets or rate limits. Without it every pod counts on its own, so a team with three pods in front of it can spend up to three times its budget before anything is refused. | + +Set `redis_host`, and optionally `redis_port` and `redis_password`, to point at an existing Redis. The module puts those values into the same secret as the model credentials and writes a `general_settings.coordination_redis` block into the proxy config that reads them from the environment. + +## Registering model backends + +`var.model_backends` is a map keyed by the alias a caller puts in the `model` field of a request. Each entry carries the name of the model at the upstream provider and the base URL of its OpenAI-compatible endpoint. The credentials live in `var.model_backend_api_keys`, keyed the same way and marked sensitive, so a plan stays readable and only the credentials are hidden. + +The module renders this into the chart's `proxy_config.model_list`: + +```yaml +model_list: + - model_name: chat-large + litellm_params: + model: openai/neuralmagic/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic + api_base: https://api.openai-compat.model-serving.eu01.onstackit.cloud/v1 + api_key: os.environ/LITELLM_API_KEY_CHAT_LARGE +``` + +Two traps are worth naming, because both produce errors that point somewhere else: + +- **The `openai/` prefix on `model` selects the OpenAI-compatible driver.** Without it LiteLLM tries to guess the provider from the model name and reaches for a different driver. The module adds the prefix, so `var.model_backends` carries the bare upstream model name. +- **`api_base` has to end with `/v1`.** LiteLLM appends the route to this URL, so an endpoint without the suffix answers "Not Found" on every call. The module rejects an `api_base` without it at plan time. + +Each alias becomes an environment variable named `LITELLM_API_KEY_`, with every character outside `[A-Za-z0-9]` replaced by an underscore. The module writes those variables into a Kubernetes Secret and lists it under `environmentSecrets`, which the chart exports into the pods with `envFrom`. That is what the `os.environ/…` references in the proxy config resolve against. Two aliases that would collapse to the same variable name are rejected at plan time, because they would otherwise share one credential. + +Several aliases can share one upstream endpoint and one credential. Repeat the value in `model_backend_api_keys` for each alias. + +## The master key + +`var.master_key` is the root credential of the gateway. It authenticates every call to the `/key` and `/team` endpoints and works as a virtual key itself. LiteLLM rejects a key that does not start with `sk-`, so the module validates the prefix. + +The module writes the key into a Kubernetes Secret and points the chart at it with `masterkeySecretName`. Without that the chart generates a key of its own on every install, which no caller knows and which changes whenever the secret is recreated. + +## Notes for platform engineers + +- **Providers.** Only `kubernetes` and `helm`. No cloud provider enters this module, so it runs on SKE, AKS and anything else that speaks the Kubernetes API. +- **Permissions.** The token in `var.token` needs to create a namespace, secrets and the workloads of the release in that namespace. Cluster-admin is not required; the chart installs no CRDs and no cluster-scoped RBAC. +- **The namespace belongs to the module.** It creates `var.namespace` and destroys it again, together with the secrets it wrote there. +- **Reachability.** The Service is a ClusterIP, so the gateway answers inside the cluster at the `api_base` output. Put an Ingress in front of it when callers live outside the cluster. + +## Usage + +```hcl +module "litellm" { + source = "github.com/meshcloud/meshstack-hub//modules/ai/litellm/buildingblock?ref=main" + + cluster_endpoint = var.cluster_endpoint + cluster_ca_certificate = var.cluster_ca_certificate + token = var.token + + master_key = var.litellm_master_key + + postgres_host = var.postgres_host + postgres_database = "litellm" + postgres_username = var.postgres_username + postgres_password = var.postgres_password + + model_backends = { + "chat-large" = { + model = "neuralmagic/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic" + api_base = "https://api.openai-compat.model-serving.eu01.onstackit.cloud/v1" + } + "embed" = { + model = "intfloat/e5-mistral-7b-instruct" + api_base = "https://api.openai-compat.model-serving.eu01.onstackit.cloud/v1" + } + } + + model_backend_api_keys = { + "chat-large" = var.stackit_model_serving_token + "embed" = var.stackit_model_serving_token + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [helm](#requirement\_helm) | >= 3.0.0 | +| [kubernetes](#requirement\_kubernetes) | >= 2.38 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.litellm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | +| [kubernetes_secret_v1.master_key](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [kubernetes_secret_v1.model_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [kubernetes_secret_v1.postgres](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chart\_version](#input\_chart\_version) | Version of the litellm-helm chart. See https://github.com/BerriAI/litellm/pkgs/container/litellm-helm. | `string` | `"1.96.2"` | no | +| [cluster\_ca\_certificate](#input\_cluster\_ca\_certificate) | Cluster CA certificate, base64 encoded. | `string` | n/a | yes | +| [cluster\_endpoint](#input\_cluster\_endpoint) | IP address or hostname of the cluster control plane, without the https:// scheme. | `string` | n/a | yes | +| [helm\_timeout](#input\_helm\_timeout) | Seconds to wait for the Helm release to become ready. The Prisma migration Job runs first and takes part of this budget. | `number` | `600` | no | +| [master\_key](#input\_master\_key) | Master key of the gateway. It must start with 'sk-', because LiteLLM rejects a key without that prefix. | `string` | n/a | yes | +| [model\_backend\_api\_keys](#input\_model\_backend\_api\_keys) | API key per model alias, keyed exactly like model\_backends. Several aliases that share one upstream endpoint repeat the same value. | `map(string)` | n/a | yes | +| [model\_backends](#input\_model\_backends) | Models the gateway exposes, keyed by the alias callers ask for in the `model` field of a request.

- `model`: name of the model at the upstream provider. The module prefixes it with `openai/`,
which is what selects the OpenAI-compatible driver.
- `api_base`: base URL of the upstream OpenAI-compatible endpoint, including the `/v1` suffix.

Pass the credential for each alias in `model_backend_api_keys` under the same key. | 
map(object({
    model    = string
    api_base = string
  }))
| n/a | yes | +| [namespace](#input\_namespace) | Namespace the gateway runs in. The module creates it. | `string` | `"litellm"` | no | +| [postgres\_database](#input\_postgres\_database) | Name of the database on the Postgres server. It has to exist before the first apply; the Prisma migration Job creates the tables inside it, not the database itself. | `string` | `"litellm"` | no | +| [postgres\_host](#input\_postgres\_host) | Hostname of the Postgres server that holds virtual keys, teams, budgets and spend records. | `string` | n/a | yes | +| [postgres\_password](#input\_postgres\_password) | Password of the Postgres user. Use only characters that are safe in a URL, because the chart substitutes the value into the connection URL without encoding it. | `string` | n/a | yes | +| [postgres\_port](#input\_postgres\_port) | Port of the Postgres server. | `number` | `5432` | no | +| [postgres\_ssl\_mode](#input\_postgres\_ssl\_mode) | Value of the sslmode parameter on the Postgres connection URL. One of 'disable', 'prefer', 'require', 'verify-ca' or 'verify-full'. | `string` | `"require"` | no | +| [postgres\_username](#input\_postgres\_username) | User the gateway connects as. It needs rights to create and alter tables, because the Prisma migration Job runs the schema migrations under this user. | `string` | n/a | yes | +| [redis\_host](#input\_redis\_host) | Hostname of an existing Redis instance the gateway coordinates through. Leave it null to run without Redis, which is only correct with a single replica. | `string` | `null` | no | +| [redis\_password](#input\_redis\_password) | Password of the Redis instance. Leave it null for a Redis without authentication. Only used when redis\_host is set. | `string` | `null` | no | +| [redis\_port](#input\_redis\_port) | Port of the Redis instance. Only used when redis\_host is set. | `number` | `6379` | no | +| [release\_name](#input\_release\_name) | Helm release name of the gateway. | `string` | `"litellm"` | no | +| [replica\_count](#input\_replica\_count) | Number of gateway pods. Set redis\_host as well when this is greater than 1. | `number` | `1` | no | +| [token](#input\_token) | Token of the service account this module runs as. It needs permission to create a namespace, secrets and the workloads of the Helm release. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_base](#output\_api\_base) | In-cluster base URL of the gateway, including the '/v1' suffix. Callers send the master key or a virtual key as a bearer token. | +| [master\_key\_secret\_name](#output\_master\_key\_secret\_name) | Name of the secret in the gateway namespace that holds the master key under the 'masterkey' key. | +| [model\_aliases](#output\_model\_aliases) | Model aliases the gateway exposes. A caller puts one of them in the 'model' field of a request. | +| [namespace](#output\_namespace) | Namespace the gateway runs in. | +| [service\_name](#output\_service\_name) | Name of the Service in front of the gateway pods. | + diff --git a/modules/ai/litellm/buildingblock/logo.png b/modules/ai/litellm/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..5b8b3f909faa35c425fc8b76dca6cbb0b28dc785 GIT binary patch literal 5905 zcmc&&S5OmNu;#~~qM(Q%U8IHH1O$Q&CMXbkk)jkK5FmjdO*%*?^p2pE z7&;=ohtPZPmwO-X(|x#eU+(VAch2nYcjnCO&d%&cYQIste~;-N5fRb-*Xqi;L`47K zZ|~9McWyBylV6>P=nj!KSYJiW6?|*Cb^jOs4j~w ztFkIMIW=~XFbm*h)6r2YsVLMm4i*-YPHXui`i#%Q)|rEY3Der8Z}1^8ul<%@@N+UZ zC)L#r3IeI=>4B5;2c@1X^8naIrJSAJ6Mjx_Ff-Enho>4^##_HnzKwUV*F8JkySh5U z6DD~iY@MA9o+)`QE&bg)SXo`2`{43+W_ECAcjfG2$IablaJb#UNekv?IyuoNBu#Cdb#--B5@)IJSUNF3 z<@zzoKQ2r0m832-EA7W*O>G4;pI(B8!gE!(Z&?w}5ol$#m#Hz%&=0WEGIV@pS4Vw{ zwBlQ<$Q-b}Z&ghaqGrh4J{gfv$R%!4P!y|e5|x`Du~hHt9h_wuTTIU?q-z-`DTC)?oR-6xL z$KkHYu0^jYuCJkDkyolzIO<9q7cQ;%uTLh~RST{^=n#yXz`m?oy#D5~61zJtIyH3> z8Cy5jJEYs>bF$^qzoGb9{pSBkuI1}(qKSwe?Yvf2)c2X(Omk6B*<$U!nv$qKTCx0m zeey$k$#Et639FZ7`LXp1@P)`XDJrrK+1zaQdYVHeO|;1|Lqb73u1_8`b#&AJ*6=~380^(a0UonA}m zuI$~=!Vw1a%7dq{)YoV$nOpAEF# zP2J}{BuJ|;Qa-R;?sBnt|7s?^>6TJLrp+tC)OUL7^K|#v>Mx)w$8@kf1uwS#*=AW+ zEJI{IWd0(LBUx9Z>L|vk>p6LtQu)}Fv!v}`fd=X0#WIWM*p>13$qfmD-%Fc}@5)=W ztZ$42zs)n8S|W`Oz85#$SBpPw>S%@``F3j42xc1B3t9Hm3wzCekV1VELH?|=nE8rQ z%>wuDt+-k|bFxRBi?UeKyKw6j{e6YDlaK7XX8}7QB6rvedba3ZkG>NK6~XXsAmI&# z5Ei*6lqV~fUJN9fkoKA&wI6u8cQJ92fxviJk~A0=o0QVM&KoWPjXR2m-eC)Tdk9NR&(<9Gf?rX-f_8xjY zpmL$)l$2SjKW;G;U^gyF#8!$Vs|u7`Vl<*0jdaBFRy!SA1F*!9z8o?fP0Pr9eU&@Y zYWnt8`YPb=1zEGY6)fVO@G-7j@!Jb zdnLA5p&qLn3CIA6C9ZZlw3=Z9AbkV4b_o5WY2baBeK3}nf&xW>!r|KIw_EGF^9QFA zB<0Wn@>tqz`VC#c=S6=i?zp*yH-9qi(Mj=B^b`~#>U6>MC|vuNd<*N7gJByiEm>cl zybr-vgYbNYqVY@gWA6KE%kutvlzY}zFyf-iCAGF7ORX$z2U;e<VR3Ob21*6^Ge*j$OHh?SYmV`cm5b*30v?2w4fYSIIi2~nQ~?~KQ`eg< z2qKLAm7Df#kTKW(MT3iHQvFb@ebjilu~9`md-3<`dUSL&r2Lyk-Xj?O1IX>q1oIhM ztOwe$mC%2BJ6o(^91l1;q5cn)&dvi`MJ#47j?9gXU(!=|@KIuEPju|STQ!@n z>Hf@`kVi;`7jW8AA~x#0Pm+jkuD6dBM-7hywygG62eU%Yv{}S3ugzqS3#)9{kj_G)Tp=htunmQ&VB))Fvh- zjg6u28#{{dZMlIC;ygRZLS+%dj%VA>C8)FJ#`Vu1T`56J6}aV-v7NBmB6gz?!|W*P z`Nu_4#A^-a{07CKPHT+f?C3Ht7uFsqpX?12Nme?v=Kj+63Ce5Z(h{`BDT7*ty&p3Q z7lb^I>K9$|%DHEVjW;A)FjUhQeK%|pBNkqt2OvuX+sFUPpV}CjODOQ@73M1#SzCU5 zmrwAFBa{VX$JNZZS$$gr_pzDdi)W%lPbQaPBgF{O~XTitz%Kmu| zj~ugeo0{A?lGkM#WVfq0Sl_R@^=&qT-Y~a}M+V(wBb{7kjE-Bbb{9&!tLlsxKRp&T z;b>r&+S!y&e?tZ>`nT%tWswWJX`MFj+V(JzLRF0O7~H#bW08<;wW0{yJg><<#K2Vr zn&O~=Kh1IV_It~wnz2gd363hNoNmvkO2U^j87}YUe4<()eP=`d>P{rOPgAu%xTMc9fPT<>D=Ql&~rx6;)nt&-Uchh+bi8&19@3V^xKEZTwSir$V8B? zl$=gOHJ1U%UG^Ozv0jGmF*)5GY-kDekZAifZf2Z%REp~D4?KsSJB6s*6`1ko!B95X zLlkN|8MDEGWKP`{IG5G#O7uGndG1<*%*iqOj^H%vdzDQ(sq;#;yI#M@J|su+*Ov)d zd&yE35<}vUANjuS?p1S#xi7g!24sIi+)U=Na6Gaq+h~|IZcpV-vuV_d2)*1A6NyeY z9scbi8yn!WO{B8$&i3rY#F`}2%=-Gm*^Md#3yWB6KPPcW7Q4G;RomQQL`Ph&4CLR! zT0Gu#-*^ZfFY%iaaNy)(*cxzFtCdyV^NRz+@CnT_p(2i{xNm3KQEL|VK#P+t(!gkk zH6BS85gy*P`D#Athb+C%yC4$=Bz(kd-_K+Phk^F*<#%Y(H;E-)7Ni6Pm`XAR60ju<_5sXxcW#=$JQs$t&b+9;KeuwREBQL!P~Qo+)1h zK&{;R_2U-npe9)xY{0A*VeD9nU|v4+@3KP*ywU9{~(6w`gNz zBjD;-S%-gUOqobd(&cJs+DuuLftC4*3X=0HIG@L=8)1wsSDzlAGLgh@bSES2F@AdDYN%w72vr z&g>8PvEel!gSADPhwl8)U4t+ONv#}+rM%xCWxte(B(h6^bFl zye()~>?`U7{=AL1@Q+zSGvg+WG25L~uSw4yQr=Y+5&d4Fg++R0$&7#PzuTJI{;)Fh zij9sY&xRKyfXT6{wDTM!QE;?!R?KoAIG8ryMw(goH{!dl+u%t_O|IMuHv@9FdXU7! zuqoak!3NaAcEsluK|BcdD4vG%UXjK~7c%tja!;y7;NIR?#y-qtDL|9!DLp{|!HyV^ zoYcwcWxjrpH8Nq{&PE+AdXy}|X50!bEHW}C>~Mf!N9~3qtG|8>|2P5N@s@&Ihf>W> zF?#v7=s7SlrQBRZZWumwixxboKwzMsgcu}F8eZ;0xIh7M!@bfEVc^e_W76Crl zsTAetJuvBExMpbZ}tEOlb#0~uW{_3+lNz0atAxyM9Rjrt$5dyHKD?YOdZCHfL*n zT|XROq;ZyIW`#>yhJYGdub5xXc0;Mo_^NJF+J_qdp!1a znZb0d5UQuOKO^DD4!o#&&W8h6|J3G#MaeCjI>e_fA9=lcQQ2f77*4E_UkM0Tr;1)$ zvsg*hjr;-^0zUWl<*Ttk0wF-3-CvR7_6Sxvzjgf?!q2`4!KJm!Xrt8B)ZG5}6^q$S zBWpXGhZhkMHKebcoMxZU0vZYjUhQY)3h?0~DxvnpG7vuBSsJdDt75<$%ps|6@a#K3 zHht%m1dtQ=20fpYJtChm(C+m#%8x_z0oju4uO&EnP4|*V(+mY)fCt8Ej5Coik!tZR z0?(O!v#gy}&Gl4ukSdq;FnUAz2Wqmzls)LeHlXP+m^>6Vvqd3}+@Lz8$KU1N2u^NsZ-kMYbaZxo(h$a8=TqXE>_hom@lh^x|`r)Ncv|0jl+ ziz3N2H|5`p$Jm{k)V?xm!LQLf&r(r=eGZBu4z(Zm$O#_q%>2{k9zLRG6I))AI$?=s z8Kz*$T3fhX!urBD;92%@nu{IhP4f%J z->fwU>32xbg3%9p40}eQFCaioKCS&(DT1aA1&zBEwo%`i^WRhf`*hy z1H9MJ5<5a}+omrJo#sWNxCmASA*KaIuoIQSXgm}id|k(7WMKui<4AtHz1FTE2Yo=9 zEN)d-SI6_{b}Ak6{Z=oHldhR`8pg}g&J<%{Py-@(C!K;ys6o7HH zkJDpgL>l63tS7=R*L&34scYixDDnhlOYo0vmujeWA@-m}=2=RJo zZn3A;wjO`spQ@KTcU5^0#J;zH80zV&Tm?0pm%%%sKjSZhX-7BBB)#64`ghb_P1CfE zdniszeLPxx@uVMtaMI){=6ku8_7gd=+Zp0iH&^ergxz7$dYJS`Q?omzKOOAE^Zcpw zSi%{9raIFaiVWtSM?Hl06?1WNaQM8lqd=Tcce|kb-^y;#^3!3Gd#>c>BuxLwd9CtB KxlGCO(|-Wj>D|Qu literal 0 HcmV?d00001 diff --git a/modules/ai/litellm/buildingblock/main.tf b/modules/ai/litellm/buildingblock/main.tf new file mode 100644 index 00000000..1e286493 --- /dev/null +++ b/modules/ai/litellm/buildingblock/main.tf @@ -0,0 +1,185 @@ +locals { + # The chart names the Service after the release: '-litellm', or just '' when + # the release name already contains the chart's nameOverride, which is 'litellm'. + service_name = strcontains(var.release_name, "litellm") ? var.release_name : "${var.release_name}-litellm" + + # Pinned here rather than left to the chart default, so the api_base output cannot drift away + # from the port the Service actually listens on. + service_port = 4000 + + # Every credential reaches the pods as an environment variable, and the proxy config refers to it + # as os.environ/. Characters an environment variable name cannot carry collapse into an + # underscore; variables.tf rejects two aliases that would collapse to the same name. + api_key_env_names = { + for alias in keys(var.model_backends) : + alias => "LITELLM_API_KEY_${upper(replace(alias, "/[^a-zA-Z0-9]/", "_"))}" + } + + redis_enabled = var.redis_host != null + + # Whether a Redis password was given is a plain fact, while the password itself is a secret. The + # fact has to stay unmarked, because everything derived from a marked value carries the mark, and + # the chart values would then be hidden in full from every plan. nonsensitive() rejects an + # argument that carries no mark, so try() falls back to the bare comparison. + redis_password_set = local.redis_enabled ? try(nonsensitive(var.redis_password != null), var.redis_password != null) : false + + redis_env = local.redis_enabled ? merge( + { + REDIS_HOST = var.redis_host + REDIS_PORT = tostring(var.redis_port) + }, + local.redis_password_set ? { REDIS_PASSWORD = var.redis_password } : {} + ) : {} + + # The proxy reads its coordination store from general_settings.coordination_redis. The chart only + # renders that block for its own bundled Redis, so an external Redis is written out here. A block + # supplied in proxy_config always wins over the chart's own. + coordination_redis = local.redis_enabled ? { + coordination_redis = merge( + { + host = "os.environ/REDIS_HOST" + port = "os.environ/REDIS_PORT" + }, + local.redis_password_set ? { password = "os.environ/REDIS_PASSWORD" } : {} + ) + } : {} + + chart_values = { + replicaCount = var.replica_count + + service = { + type = "ClusterIP" + port = local.service_port + } + + # Without this the chart generates a master key of its own on every install, which no caller + # knows and which changes whenever the secret is recreated. + masterkeySecretName = kubernetes_secret_v1.master_key.metadata[0].name + masterkeySecretKey = "masterkey" + + # Exports the secret into the pods as environment variables, which is what the + # os.environ/ references in proxy_config resolve against. + environmentSecrets = [kubernetes_secret_v1.model_credentials.metadata[0].name] + + db = { + # The chart bundles a Bitnami postgresql subchart and turns it on by default. Those images + # no longer receive updates and the subchart pins bitnamilegacy/postgresql, so the database + # comes from outside the chart. + deployStandalone = false + useExisting = true + + endpoint = var.postgres_host + database = var.postgres_database + + # Kubernetes substitutes $(VAR) from the environment variables declared before this one in + # the same container, so the credentials stay in the secret and never appear in the pod spec. + # The chart's default URL carries no port, hence the override. + url = "postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST):${var.postgres_port}/$(DATABASE_NAME)?sslmode=${var.postgres_ssl_mode}" + + secret = { + name = kubernetes_secret_v1.postgres.metadata[0].name + usernameKey = "username" + passwordKey = "password" + } + } + + # The bundled Redis subchart carries the same retired Bitnami images as the Postgres one, so it + # stays off. An external Redis arrives through REDIS_HOST, REDIS_PORT and REDIS_PASSWORD. + redis = { + enabled = false + } + + migrationJob = { + enabled = true + hooks = { + # As a Helm pre-install and pre-upgrade hook the Job runs to completion before the + # Deployment is created, so the pods never start against a database without the schema. + # The chart's default instead annotates the Job for ArgoCD, which means nothing here. + helm = { enabled = true } + argocd = { enabled = false } + } + } + + proxy_config = { + model_list = [ + for alias, backend in var.model_backends : { + model_name = alias + litellm_params = { + # The 'openai/' prefix selects the OpenAI-compatible driver. + model = "openai/${backend.model}" + # api_base carries the '/v1' suffix; variables.tf rejects an endpoint without it. + api_base = backend.api_base + api_key = "os.environ/${local.api_key_env_names[alias]}" + } + } + ] + + general_settings = merge( + { master_key = "os.environ/PROXY_MASTER_KEY" }, + local.coordination_redis + ) + } + } +} + +resource "kubernetes_namespace_v1" "this" { + metadata { + name = var.namespace + } +} + +resource "kubernetes_secret_v1" "master_key" { + metadata { + name = "${var.release_name}-master-key" + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + + data = { + masterkey = var.master_key + } +} + +resource "kubernetes_secret_v1" "postgres" { + metadata { + name = "${var.release_name}-postgres" + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + + data = { + username = var.postgres_username + password = var.postgres_password + } +} + +resource "kubernetes_secret_v1" "model_credentials" { + metadata { + name = "${var.release_name}-model-credentials" + namespace = kubernetes_namespace_v1.this.metadata[0].name + } + + data = merge( + { + for alias, env_name in local.api_key_env_names : + env_name => var.model_backend_api_keys[alias] + }, + local.redis_env + ) +} + +# The chart is published to GHCR as an OCI artifact and has no classic Helm repository. The helm +# provider takes the registry and the path prefix as the repository, and the chart name on its own +# as the chart, so the reference resolves to oci://ghcr.io/berriai/litellm-helm. +resource "helm_release" "litellm" { + name = var.release_name + namespace = kubernetes_namespace_v1.this.metadata[0].name + repository = "oci://ghcr.io/berriai" + chart = "litellm-helm" + version = var.chart_version + + create_namespace = false + atomic = true + wait = true + timeout = var.helm_timeout + + values = [yamlencode(local.chart_values)] +} diff --git a/modules/ai/litellm/buildingblock/outputs.tf b/modules/ai/litellm/buildingblock/outputs.tf new file mode 100644 index 00000000..781d9c0b --- /dev/null +++ b/modules/ai/litellm/buildingblock/outputs.tf @@ -0,0 +1,30 @@ +output "namespace" { + description = "Namespace the gateway runs in." + value = kubernetes_namespace_v1.this.metadata[0].name +} + +output "service_name" { + description = "Name of the Service in front of the gateway pods." + value = local.service_name + + depends_on = [helm_release.litellm] +} + +output "api_base" { + # The '/v1' suffix belongs to the base URL. A client that drops it gets a 'Not Found' error, so + # this output carries the suffix instead of leaving it to the caller. + description = "In-cluster base URL of the gateway, including the '/v1' suffix. Callers send the master key or a virtual key as a bearer token." + value = "http://${local.service_name}.${kubernetes_namespace_v1.this.metadata[0].name}.svc.cluster.local:${local.service_port}/v1" + + depends_on = [helm_release.litellm] +} + +output "model_aliases" { + description = "Model aliases the gateway exposes. A caller puts one of them in the 'model' field of a request." + value = sort(keys(var.model_backends)) +} + +output "master_key_secret_name" { + description = "Name of the secret in the gateway namespace that holds the master key under the 'masterkey' key." + value = kubernetes_secret_v1.master_key.metadata[0].name +} diff --git a/modules/ai/litellm/buildingblock/provider.tf b/modules/ai/litellm/buildingblock/provider.tf new file mode 100644 index 00000000..4a86315b --- /dev/null +++ b/modules/ai/litellm/buildingblock/provider.tf @@ -0,0 +1,15 @@ +provider "kubernetes" { + host = "https://${var.cluster_endpoint}" + cluster_ca_certificate = base64decode(var.cluster_ca_certificate) + token = var.token +} + +# The helm provider talks to the same control plane with the same credentials, so the namespace, +# the secrets and the Helm release all land in one cluster without extra wiring. +provider "helm" { + kubernetes = { + host = "https://${var.cluster_endpoint}" + cluster_ca_certificate = base64decode(var.cluster_ca_certificate) + token = var.token + } +} diff --git a/modules/ai/litellm/buildingblock/variables.tf b/modules/ai/litellm/buildingblock/variables.tf new file mode 100644 index 00000000..22486683 --- /dev/null +++ b/modules/ai/litellm/buildingblock/variables.tf @@ -0,0 +1,183 @@ +variable "cluster_endpoint" { + type = string + description = "IP address or hostname of the cluster control plane, without the https:// scheme." +} + +variable "cluster_ca_certificate" { + type = string + description = "Cluster CA certificate, base64 encoded." +} + +variable "token" { + type = string + sensitive = true + description = "Token of the service account this module runs as. It needs permission to create a namespace, secrets and the workloads of the Helm release." +} + +variable "namespace" { + type = string + default = "litellm" + description = "Namespace the gateway runs in. The module creates it." +} + +variable "release_name" { + type = string + default = "litellm" + # The chart derives the Service name from the release name: '-litellm', or just + # '' when the release name already contains 'litellm'. + description = "Helm release name of the gateway." +} + +variable "chart_version" { + type = string + default = "1.96.2" + # The chart is published to GHCR as an OCI artifact only, so the tag on the registry is the + # single source of truth. Chart.yaml on the repository's main branch reads a different, much + # lower version, because the release pipeline overwrites it while publishing. + description = "Version of the litellm-helm chart. See https://github.com/BerriAI/litellm/pkgs/container/litellm-helm." +} + +variable "replica_count" { + type = number + default = 1 + # Anything above 1 needs Redis, otherwise each pod counts rate limits and spend on its own. + description = "Number of gateway pods. Set redis_host as well when this is greater than 1." +} + +variable "helm_timeout" { + type = number + default = 600 + description = "Seconds to wait for the Helm release to become ready. The Prisma migration Job runs first and takes part of this budget." +} + +variable "master_key" { + type = string + sensitive = true + # The proxy reads it as PROXY_MASTER_KEY and treats it as the root credential of the gateway: + # it authenticates every call to the /key and /team endpoints and works as a virtual key itself. + description = "Master key of the gateway. It must start with 'sk-', because LiteLLM rejects a key without that prefix." + + validation { + condition = startswith(var.master_key, "sk-") + error_message = "The master key must start with 'sk-'." + } +} + +variable "model_backends" { + type = map(object({ + model = string + api_base = string + })) + description = <<-EOT + Models the gateway exposes, keyed by the alias callers ask for in the `model` field of a request. + + - `model`: name of the model at the upstream provider. The module prefixes it with `openai/`, + which is what selects the OpenAI-compatible driver. + - `api_base`: base URL of the upstream OpenAI-compatible endpoint, including the `/v1` suffix. + + Pass the credential for each alias in `model_backend_api_keys` under the same key. + EOT + + validation { + condition = length(var.model_backends) > 0 + error_message = "Register at least one model backend. The gateway refuses to start with an empty model list." + } + + validation { + condition = alltrue([for backend in var.model_backends : endswith(backend.api_base, "/v1")]) + error_message = "Every api_base must end with '/v1'. LiteLLM appends the route to this URL, so an endpoint without the suffix answers 'Not Found'." + } + + validation { + # Each alias becomes an environment variable name, and everything outside [A-Za-z0-9] collapses + # into an underscore on the way. Two aliases that collapse to the same name would share one + # credential, so reject that pair here instead of routing a request with the wrong key. + condition = length(distinct([ + for alias in keys(var.model_backends) : upper(replace(alias, "/[^a-zA-Z0-9]/", "_")) + ])) == length(var.model_backends) + error_message = "Two aliases differ only in characters outside [A-Za-z0-9] and would map to the same environment variable name. Rename one of them." + } +} + +variable "model_backend_api_keys" { + type = map(string) + sensitive = true + # Kept out of var.model_backends so the backend map stays readable in plan output and in the + # meshStack UI, and so only the credentials carry the sensitivity mark. + description = "API key per model alias, keyed exactly like model_backends. Several aliases that share one upstream endpoint repeat the same value." + + validation { + condition = alltrue([ + for alias in keys(var.model_backends) : + contains(try(nonsensitive(keys(var.model_backend_api_keys)), keys(var.model_backend_api_keys)), alias) + ]) + error_message = "Every key in model_backends needs an entry with the same key in model_backend_api_keys." + } +} + +variable "postgres_host" { + type = string + # Virtual keys, teams, budgets and spend tracking all live in Postgres. Without a database the + # gateway is a stateless proxy and none of those endpoints work, so the database is required. + description = "Hostname of the Postgres server that holds virtual keys, teams, budgets and spend records." +} + +variable "postgres_port" { + type = number + default = 5432 + description = "Port of the Postgres server." +} + +variable "postgres_database" { + type = string + default = "litellm" + description = "Name of the database on the Postgres server. It has to exist before the first apply; the Prisma migration Job creates the tables inside it, not the database itself." +} + +variable "postgres_username" { + type = string + description = "User the gateway connects as. It needs rights to create and alter tables, because the Prisma migration Job runs the schema migrations under this user." +} + +variable "postgres_password" { + type = string + sensitive = true + # The chart builds the connection URL from $(DATABASE_USERNAME) and $(DATABASE_PASSWORD), which + # Kubernetes substitutes verbatim and does not URL-encode. A password containing ':', '@', '/' + # or '?' therefore breaks the URL. + description = "Password of the Postgres user. Use only characters that are safe in a URL, because the chart substitutes the value into the connection URL without encoding it." +} + +variable "postgres_ssl_mode" { + type = string + default = "require" + # Managed Postgres offerings terminate TLS, so requiring it is the safe default. A server + # without TLS needs 'prefer' or 'disable'. + description = "Value of the sslmode parameter on the Postgres connection URL. One of 'disable', 'prefer', 'require', 'verify-ca' or 'verify-full'." + + validation { + condition = contains(["disable", "prefer", "require", "verify-ca", "verify-full"], var.postgres_ssl_mode) + error_message = "postgres_ssl_mode must be one of 'disable', 'prefer', 'require', 'verify-ca' or 'verify-full'." + } +} + +variable "redis_host" { + type = string + default = null + # Redis is the coordination store of the gateway: cross-pod rate limits, spend tracking and the + # pod lock manager. One pod needs none of that, several pods do. + description = "Hostname of an existing Redis instance the gateway coordinates through. Leave it null to run without Redis, which is only correct with a single replica." +} + +variable "redis_port" { + type = number + default = 6379 + description = "Port of the Redis instance. Only used when redis_host is set." +} + +variable "redis_password" { + type = string + sensitive = true + default = null + description = "Password of the Redis instance. Leave it null for a Redis without authentication. Only used when redis_host is set." +} diff --git a/modules/ai/litellm/buildingblock/versions.tf b/modules/ai/litellm/buildingblock/versions.tf new file mode 100644 index 00000000..cc6c09d8 --- /dev/null +++ b/modules/ai/litellm/buildingblock/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + helm = { + source = "hashicorp/helm" + # The helm provider takes its cluster credentials as the `kubernetes = {}` attribute + # starting with 3.0.0. Earlier versions expect a `kubernetes {}` block instead. + version = ">= 3.0.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.38" + } + } +} From 08108c60bb5757d5573ac34ed910e6ce76bd0515 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:36:25 +0200 Subject: [PATCH 24/60] feat(ai/litellm-team): add the tenant-facing LiteLLM team building block Creates one LiteLLM team per tenant with a budget, and one virtual key scoped to that team. The building block definition is TENANT_LEVEL and marked use_in_landing_zones_only, so an AI landing zone lists it in spec.mandatory_building_block_refs and meshStack provisions the team when the tenant is created. Every input is either STATIC or filled from the meshStack tenant context, so tenant creation needs no form. The team_id output is assigned as PLATFORM_TENANT_ID, which lets building blocks ordered later bind their resources to the tenant's team. The ncecere/litellm provider is pinned to exactly 2.0.1, a deliberate exception to the hub rule that provider constraints use '>='. versions.tf carries the reason. Co-Authored-By: Claude Opus 5 (1M context) --- .../ai/litellm-team/buildingblock/README.md | 77 +++++ .../buildingblock/SUMMARY.md.tftpl | 27 ++ .../ai/litellm-team/buildingblock/logo.png | Bin 0 -> 5905 bytes modules/ai/litellm-team/buildingblock/main.tf | 34 +++ .../ai/litellm-team/buildingblock/outputs.tf | 39 +++ .../ai/litellm-team/buildingblock/provider.tf | 4 + .../litellm-team/buildingblock/variables.tf | 56 ++++ .../ai/litellm-team/buildingblock/versions.tf | 16 + .../ai/litellm-team/meshstack_integration.tf | 280 ++++++++++++++++++ 9 files changed, 533 insertions(+) create mode 100644 modules/ai/litellm-team/buildingblock/README.md create mode 100644 modules/ai/litellm-team/buildingblock/SUMMARY.md.tftpl create mode 100644 modules/ai/litellm-team/buildingblock/logo.png create mode 100644 modules/ai/litellm-team/buildingblock/main.tf create mode 100644 modules/ai/litellm-team/buildingblock/outputs.tf create mode 100644 modules/ai/litellm-team/buildingblock/provider.tf create mode 100644 modules/ai/litellm-team/buildingblock/variables.tf create mode 100644 modules/ai/litellm-team/buildingblock/versions.tf create mode 100644 modules/ai/litellm-team/meshstack_integration.tf diff --git a/modules/ai/litellm-team/buildingblock/README.md b/modules/ai/litellm-team/buildingblock/README.md new file mode 100644 index 00000000..268f1b56 --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/README.md @@ -0,0 +1,77 @@ +--- +name: LiteLLM Team +supportedPlatforms: + - ai +description: Creates the LiteLLM team of a tenant with a budget, and a virtual key scoped to that team. +# The module talks only to the LiteLLM admin API and receives the gateway URL and the admin key as +# static inputs, so there is no cloud-side setup to perform ahead of time. +requiresBackplane: false +--- + +# LiteLLM Team Building Block + +This building block creates one LiteLLM team per tenant and one virtual key scoped to that team. +meshStack provisions it when a tenant in the AI landing zone is created, because the AI landing zone +lists the building block definition in `spec.mandatory_building_block_refs`. The application team +fills in nothing. + +The team carries the budget. LiteLLM counts the spend of every key of a team against the team +budget, so one limit covers the tenant even when the platform team hands out a second key later. +The virtual key therefore sets no budget of its own. + +The key omits `models` on purpose. The provider sends `all-team-models` to LiteLLM when `team_id` +is set and `models` is left out, so the model allow-list stays on the team alone and a landing zone +change reaches the key without recreating it. + +LiteLLM returns the virtual key once, at creation, and never again. The `virtual_key` output is +marked sensitive and the `summary` output carries the key as well, so the value is available to the +application team on the building block run and nowhere else. + +The `ncecere/litellm` provider is pinned to exactly `2.0.1`. This is a deliberate exception to the +hub rule that provider constraints use `>=`, and `versions.tf` explains why. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [litellm](#requirement\_litellm) | = 2.0.1 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [litellm_key.this](https://registry.terraform.io/providers/ncecere/litellm/2.0.1/docs/resources/key) | resource | +| [litellm_team.this](https://registry.terraform.io/providers/ncecere/litellm/2.0.1/docs/resources/team) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [budget\_duration](#input\_budget\_duration) | Length of one budget period, after which LiteLLM resets the spend counter. Written as a LiteLLM duration such as '30d', '7d' or '1h'. | `string` | `"30d"` | no | +| [key\_alias](#input\_key\_alias) | Alias of the virtual key. Leave unset to use '-key'. | `string` | `null` | no | +| [litellm\_api\_base](#input\_litellm\_api\_base) | Base URL of the LiteLLM gateway, for example 'https://litellm.example.com'. The provider talks to the admin API under this URL. | `string` | n/a | yes | +| [litellm\_api\_key](#input\_litellm\_api\_key) | LiteLLM admin key the provider authenticates with. It needs permission to create teams and keys. | `string` | n/a | yes | +| [max\_budget](#input\_max\_budget) | Spending limit of the team for one budget period, in the currency the gateway reports spend in. LiteLLM blocks the team once the limit is reached. | `number` | `100` | no | +| [meshstack\_tenant\_uuid](#input\_meshstack\_tenant\_uuid) | UUID of the meshStack tenant. It is written to the team metadata so an operator can trace a LiteLLM team back to its tenant. | `string` | `""` | no | +| [models](#input\_models) | Names of the models on the LiteLLM gateway that the team may call. An empty list sends no allow-list to LiteLLM. | `list(string)` | `[]` | no | +| [project\_identifier](#input\_project\_identifier) | Identifier of the meshStack project the tenant belongs to. It becomes part of the team alias and is written to the team metadata. | `string` | n/a | yes | +| [team\_alias](#input\_team\_alias) | Alias of the LiteLLM team. Leave unset to use '.'. | `string` | `null` | no | +| [workspace\_identifier](#input\_workspace\_identifier) | Identifier of the meshStack workspace the tenant belongs to. It becomes part of the team alias and is written to the team metadata. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_base](#output\_api\_base) | OpenAI-compatible base URL of the LiteLLM gateway, including the '/v1' suffix. | +| [key\_id](#output\_key\_id) | Hash of the virtual key. LiteLLM identifies the key by this value, and it is safe to show in logs. | +| [summary](#output\_summary) | Summary with the endpoint, the virtual key and the budget. | +| [team\_alias](#output\_team\_alias) | Alias of the LiteLLM team, shown in the LiteLLM UI. | +| [team\_id](#output\_team\_id) | ID of the LiteLLM team. meshStack uses it as the platform tenant ID, so later building blocks can bind resources to this team. | +| [virtual\_key](#output\_virtual\_key) | The virtual key the application sends as a bearer token. LiteLLM returns it once, at creation, and never again. | + diff --git a/modules/ai/litellm-team/buildingblock/SUMMARY.md.tftpl b/modules/ai/litellm-team/buildingblock/SUMMARY.md.tftpl new file mode 100644 index 00000000..ccd93eb7 --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/SUMMARY.md.tftpl @@ -0,0 +1,27 @@ +# LiteLLM Team: **${team_alias}** + +## Details + +| Property | Value | +|----------|-------| +| **API Base URL** | `${api_base}` | +| **Virtual Key** | `${virtual_key}` | +| **Team ID** | `${team_id}` | +| **Budget** | ${max_budget} per ${budget_duration} | +| **Allowed models** | ${length(models) > 0 ? join(", ", models) : "set by the platform team on the gateway"} | + +## Calling the gateway + +The gateway speaks the OpenAI API and the base URL ends in `/v1`. Send the virtual key as a bearer +token in the `Authorization` header: + +```sh +curl "${api_base}/models" \ + -H "Authorization: Bearer ${virtual_key}" +``` + +Point an OpenAI client library at the base URL and pass the virtual key as the API key. LiteLLM +returns the key once, when the building block runs, so store it in your own secret store. + +LiteLLM stops answering once the team reaches its budget for the current period, and the spend +counter resets at the end of every period. diff --git a/modules/ai/litellm-team/buildingblock/logo.png b/modules/ai/litellm-team/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..5b8b3f909faa35c425fc8b76dca6cbb0b28dc785 GIT binary patch literal 5905 zcmc&&S5OmNu;#~~qM(Q%U8IHH1O$Q&CMXbkk)jkK5FmjdO*%*?^p2pE z7&;=ohtPZPmwO-X(|x#eU+(VAch2nYcjnCO&d%&cYQIste~;-N5fRb-*Xqi;L`47K zZ|~9McWyBylV6>P=nj!KSYJiW6?|*Cb^jOs4j~w ztFkIMIW=~XFbm*h)6r2YsVLMm4i*-YPHXui`i#%Q)|rEY3Der8Z}1^8ul<%@@N+UZ zC)L#r3IeI=>4B5;2c@1X^8naIrJSAJ6Mjx_Ff-Enho>4^##_HnzKwUV*F8JkySh5U z6DD~iY@MA9o+)`QE&bg)SXo`2`{43+W_ECAcjfG2$IablaJb#UNekv?IyuoNBu#Cdb#--B5@)IJSUNF3 z<@zzoKQ2r0m832-EA7W*O>G4;pI(B8!gE!(Z&?w}5ol$#m#Hz%&=0WEGIV@pS4Vw{ zwBlQ<$Q-b}Z&ghaqGrh4J{gfv$R%!4P!y|e5|x`Du~hHt9h_wuTTIU?q-z-`DTC)?oR-6xL z$KkHYu0^jYuCJkDkyolzIO<9q7cQ;%uTLh~RST{^=n#yXz`m?oy#D5~61zJtIyH3> z8Cy5jJEYs>bF$^qzoGb9{pSBkuI1}(qKSwe?Yvf2)c2X(Omk6B*<$U!nv$qKTCx0m zeey$k$#Et639FZ7`LXp1@P)`XDJrrK+1zaQdYVHeO|;1|Lqb73u1_8`b#&AJ*6=~380^(a0UonA}m zuI$~=!Vw1a%7dq{)YoV$nOpAEF# zP2J}{BuJ|;Qa-R;?sBnt|7s?^>6TJLrp+tC)OUL7^K|#v>Mx)w$8@kf1uwS#*=AW+ zEJI{IWd0(LBUx9Z>L|vk>p6LtQu)}Fv!v}`fd=X0#WIWM*p>13$qfmD-%Fc}@5)=W ztZ$42zs)n8S|W`Oz85#$SBpPw>S%@``F3j42xc1B3t9Hm3wzCekV1VELH?|=nE8rQ z%>wuDt+-k|bFxRBi?UeKyKw6j{e6YDlaK7XX8}7QB6rvedba3ZkG>NK6~XXsAmI&# z5Ei*6lqV~fUJN9fkoKA&wI6u8cQJ92fxviJk~A0=o0QVM&KoWPjXR2m-eC)Tdk9NR&(<9Gf?rX-f_8xjY zpmL$)l$2SjKW;G;U^gyF#8!$Vs|u7`Vl<*0jdaBFRy!SA1F*!9z8o?fP0Pr9eU&@Y zYWnt8`YPb=1zEGY6)fVO@G-7j@!Jb zdnLA5p&qLn3CIA6C9ZZlw3=Z9AbkV4b_o5WY2baBeK3}nf&xW>!r|KIw_EGF^9QFA zB<0Wn@>tqz`VC#c=S6=i?zp*yH-9qi(Mj=B^b`~#>U6>MC|vuNd<*N7gJByiEm>cl zybr-vgYbNYqVY@gWA6KE%kutvlzY}zFyf-iCAGF7ORX$z2U;e<VR3Ob21*6^Ge*j$OHh?SYmV`cm5b*30v?2w4fYSIIi2~nQ~?~KQ`eg< z2qKLAm7Df#kTKW(MT3iHQvFb@ebjilu~9`md-3<`dUSL&r2Lyk-Xj?O1IX>q1oIhM ztOwe$mC%2BJ6o(^91l1;q5cn)&dvi`MJ#47j?9gXU(!=|@KIuEPju|STQ!@n z>Hf@`kVi;`7jW8AA~x#0Pm+jkuD6dBM-7hywygG62eU%Yv{}S3ugzqS3#)9{kj_G)Tp=htunmQ&VB))Fvh- zjg6u28#{{dZMlIC;ygRZLS+%dj%VA>C8)FJ#`Vu1T`56J6}aV-v7NBmB6gz?!|W*P z`Nu_4#A^-a{07CKPHT+f?C3Ht7uFsqpX?12Nme?v=Kj+63Ce5Z(h{`BDT7*ty&p3Q z7lb^I>K9$|%DHEVjW;A)FjUhQeK%|pBNkqt2OvuX+sFUPpV}CjODOQ@73M1#SzCU5 zmrwAFBa{VX$JNZZS$$gr_pzDdi)W%lPbQaPBgF{O~XTitz%Kmu| zj~ugeo0{A?lGkM#WVfq0Sl_R@^=&qT-Y~a}M+V(wBb{7kjE-Bbb{9&!tLlsxKRp&T z;b>r&+S!y&e?tZ>`nT%tWswWJX`MFj+V(JzLRF0O7~H#bW08<;wW0{yJg><<#K2Vr zn&O~=Kh1IV_It~wnz2gd363hNoNmvkO2U^j87}YUe4<()eP=`d>P{rOPgAu%xTMc9fPT<>D=Ql&~rx6;)nt&-Uchh+bi8&19@3V^xKEZTwSir$V8B? zl$=gOHJ1U%UG^Ozv0jGmF*)5GY-kDekZAifZf2Z%REp~D4?KsSJB6s*6`1ko!B95X zLlkN|8MDEGWKP`{IG5G#O7uGndG1<*%*iqOj^H%vdzDQ(sq;#;yI#M@J|su+*Ov)d zd&yE35<}vUANjuS?p1S#xi7g!24sIi+)U=Na6Gaq+h~|IZcpV-vuV_d2)*1A6NyeY z9scbi8yn!WO{B8$&i3rY#F`}2%=-Gm*^Md#3yWB6KPPcW7Q4G;RomQQL`Ph&4CLR! zT0Gu#-*^ZfFY%iaaNy)(*cxzFtCdyV^NRz+@CnT_p(2i{xNm3KQEL|VK#P+t(!gkk zH6BS85gy*P`D#Athb+C%yC4$=Bz(kd-_K+Phk^F*<#%Y(H;E-)7Ni6Pm`XAR60ju<_5sXxcW#=$JQs$t&b+9;KeuwREBQL!P~Qo+)1h zK&{;R_2U-npe9)xY{0A*VeD9nU|v4+@3KP*ywU9{~(6w`gNz zBjD;-S%-gUOqobd(&cJs+DuuLftC4*3X=0HIG@L=8)1wsSDzlAGLgh@bSES2F@AdDYN%w72vr z&g>8PvEel!gSADPhwl8)U4t+ONv#}+rM%xCWxte(B(h6^bFl zye()~>?`U7{=AL1@Q+zSGvg+WG25L~uSw4yQr=Y+5&d4Fg++R0$&7#PzuTJI{;)Fh zij9sY&xRKyfXT6{wDTM!QE;?!R?KoAIG8ryMw(goH{!dl+u%t_O|IMuHv@9FdXU7! zuqoak!3NaAcEsluK|BcdD4vG%UXjK~7c%tja!;y7;NIR?#y-qtDL|9!DLp{|!HyV^ zoYcwcWxjrpH8Nq{&PE+AdXy}|X50!bEHW}C>~Mf!N9~3qtG|8>|2P5N@s@&Ihf>W> zF?#v7=s7SlrQBRZZWumwixxboKwzMsgcu}F8eZ;0xIh7M!@bfEVc^e_W76Crl zsTAetJuvBExMpbZ}tEOlb#0~uW{_3+lNz0atAxyM9Rjrt$5dyHKD?YOdZCHfL*n zT|XROq;ZyIW`#>yhJYGdub5xXc0;Mo_^NJF+J_qdp!1a znZb0d5UQuOKO^DD4!o#&&W8h6|J3G#MaeCjI>e_fA9=lcQQ2f77*4E_UkM0Tr;1)$ zvsg*hjr;-^0zUWl<*Ttk0wF-3-CvR7_6Sxvzjgf?!q2`4!KJm!Xrt8B)ZG5}6^q$S zBWpXGhZhkMHKebcoMxZU0vZYjUhQY)3h?0~DxvnpG7vuBSsJdDt75<$%ps|6@a#K3 zHht%m1dtQ=20fpYJtChm(C+m#%8x_z0oju4uO&EnP4|*V(+mY)fCt8Ej5Coik!tZR z0?(O!v#gy}&Gl4ukSdq;FnUAz2Wqmzls)LeHlXP+m^>6Vvqd3}+@Lz8$KU1N2u^NsZ-kMYbaZxo(h$a8=TqXE>_hom@lh^x|`r)Ncv|0jl+ ziz3N2H|5`p$Jm{k)V?xm!LQLf&r(r=eGZBu4z(Zm$O#_q%>2{k9zLRG6I))AI$?=s z8Kz*$T3fhX!urBD;92%@nu{IhP4f%J z->fwU>32xbg3%9p40}eQFCaioKCS&(DT1aA1&zBEwo%`i^WRhf`*hy z1H9MJ5<5a}+omrJo#sWNxCmASA*KaIuoIQSXgm}id|k(7WMKui<4AtHz1FTE2Yo=9 zEN)d-SI6_{b}Ak6{Z=oHldhR`8pg}g&J<%{Py-@(C!K;ys6o7HH zkJDpgL>l63tS7=R*L&34scYixDDnhlOYo0vmujeWA@-m}=2=RJo zZn3A;wjO`spQ@KTcU5^0#J;zH80zV&Tm?0pm%%%sKjSZhX-7BBB)#64`ghb_P1CfE zdniszeLPxx@uVMtaMI){=6ku8_7gd=+Zp0iH&^ergxz7$dYJS`Q?omzKOOAE^Zcpw zSi%{9raIFaiVWtSM?Hl06?1WNaQM8lqd=Tcce|kb-^y;#^3!3Gd#>c>BuxLwd9CtB KxlGCO(|-Wj>D|Qu literal 0 HcmV?d00001 diff --git a/modules/ai/litellm-team/buildingblock/main.tf b/modules/ai/litellm-team/buildingblock/main.tf new file mode 100644 index 00000000..53cd14a4 --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/main.tf @@ -0,0 +1,34 @@ +locals { + # Two projects in different workspaces can carry the same project identifier, so the team alias + # joins both identifiers. The pair is unique in meshStack and stays readable in the LiteLLM UI. + team_alias = coalesce(var.team_alias, "${var.workspace_identifier}.${var.project_identifier}") + key_alias = coalesce(var.key_alias, "${local.team_alias}-key") + + # LiteLLM's OpenAI-compatible routes live under '/v1'. The gateway also answers without the + # prefix, but OpenAI client libraries expect it, so the output carries it. + api_base = "${trimsuffix(var.litellm_api_base, "/")}/v1" +} + +resource "litellm_team" "this" { + team_alias = local.team_alias + models = var.models + + # The budget belongs on the team rather than on the key. LiteLLM counts the spend of every key + # of a team against the team budget, so a single limit here covers the tenant even if the + # platform team later hands out a second key. + max_budget = var.max_budget + budget_duration = var.budget_duration + + metadata = { + meshstack_workspace_identifier = var.workspace_identifier + meshstack_project_identifier = var.project_identifier + meshstack_tenant_uuid = var.meshstack_tenant_uuid + } +} + +# The key deliberately omits `models`: the provider sends "all-team-models" when `team_id` is set +# and `models` is left out, which keeps the allow-list on the team alone. +resource "litellm_key" "this" { + key_alias = local.key_alias + team_id = litellm_team.this.id +} diff --git a/modules/ai/litellm-team/buildingblock/outputs.tf b/modules/ai/litellm-team/buildingblock/outputs.tf new file mode 100644 index 00000000..7ce4d685 --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/outputs.tf @@ -0,0 +1,39 @@ +output "team_id" { + value = litellm_team.this.id + description = "ID of the LiteLLM team. meshStack uses it as the platform tenant ID, so later building blocks can bind resources to this team." +} + +output "team_alias" { + value = litellm_team.this.team_alias + description = "Alias of the LiteLLM team, shown in the LiteLLM UI." +} + +output "virtual_key" { + value = litellm_key.this.key + sensitive = true + description = "The virtual key the application sends as a bearer token. LiteLLM returns it once, at creation, and never again." +} + +output "key_id" { + value = litellm_key.this.id + description = "Hash of the virtual key. LiteLLM identifies the key by this value, and it is safe to show in logs." +} + +output "api_base" { + value = local.api_base + description = "OpenAI-compatible base URL of the LiteLLM gateway, including the '/v1' suffix." +} + +output "summary" { + description = "Summary with the endpoint, the virtual key and the budget." + sensitive = true + value = templatefile("${path.module}/SUMMARY.md.tftpl", { + team_alias = litellm_team.this.team_alias + team_id = litellm_team.this.id + virtual_key = litellm_key.this.key + api_base = local.api_base + max_budget = var.max_budget + budget_duration = var.budget_duration + models = var.models + }) +} diff --git a/modules/ai/litellm-team/buildingblock/provider.tf b/modules/ai/litellm-team/buildingblock/provider.tf new file mode 100644 index 00000000..7f3b4858 --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/provider.tf @@ -0,0 +1,4 @@ +provider "litellm" { + api_base = var.litellm_api_base + api_key = var.litellm_api_key +} diff --git a/modules/ai/litellm-team/buildingblock/variables.tf b/modules/ai/litellm-team/buildingblock/variables.tf new file mode 100644 index 00000000..ea93a29a --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/variables.tf @@ -0,0 +1,56 @@ +variable "litellm_api_base" { + type = string + description = "Base URL of the LiteLLM gateway, for example 'https://litellm.example.com'. The provider talks to the admin API under this URL." +} + +variable "litellm_api_key" { + type = string + sensitive = true + description = "LiteLLM admin key the provider authenticates with. It needs permission to create teams and keys." +} + +variable "workspace_identifier" { + type = string + description = "Identifier of the meshStack workspace the tenant belongs to. It becomes part of the team alias and is written to the team metadata." +} + +variable "project_identifier" { + type = string + description = "Identifier of the meshStack project the tenant belongs to. It becomes part of the team alias and is written to the team metadata." +} + +variable "meshstack_tenant_uuid" { + type = string + default = "" + description = "UUID of the meshStack tenant. It is written to the team metadata so an operator can trace a LiteLLM team back to its tenant." +} + +variable "team_alias" { + type = string + default = null + description = "Alias of the LiteLLM team. Leave unset to use '.'." +} + +variable "key_alias" { + type = string + default = null + description = "Alias of the virtual key. Leave unset to use '-key'." +} + +variable "models" { + type = list(string) + default = [] + description = "Names of the models on the LiteLLM gateway that the team may call. An empty list sends no allow-list to LiteLLM." +} + +variable "max_budget" { + type = number + default = 100 + description = "Spending limit of the team for one budget period, in the currency the gateway reports spend in. LiteLLM blocks the team once the limit is reached." +} + +variable "budget_duration" { + type = string + default = "30d" + description = "Length of one budget period, after which LiteLLM resets the spend counter. Written as a LiteLLM duration such as '30d', '7d' or '1h'." +} diff --git a/modules/ai/litellm-team/buildingblock/versions.tf b/modules/ai/litellm-team/buildingblock/versions.tf new file mode 100644 index 00000000..a3bbc436 --- /dev/null +++ b/modules/ai/litellm-team/buildingblock/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + litellm = { + source = "ncecere/litellm" + # Exact pin, a deliberate exception to the hub rule that provider constraints use '>='. + # ncecere/litellm is a community provider with a single maintainer, and it has changed + # resource behaviour inside a minor release before: v1.2.0 replaced the id of litellm_key + # with a hash of the key. LiteLLM returns a virtual key once and never again, so a provider + # change that recreates the key takes the credential away from a running application. The + # version is therefore pinned here and raised deliberately after a review of the changelog. + version = "= 2.0.1" + } + } +} diff --git a/modules/ai/litellm-team/meshstack_integration.tf b/modules/ai/litellm-team/meshstack_integration.tf new file mode 100644 index 00000000..f7754709 --- /dev/null +++ b/modules/ai/litellm-team/meshstack_integration.tf @@ -0,0 +1,280 @@ +variable "litellm_api_base" { + type = string + description = "Base URL of the LiteLLM gateway, for example 'https://litellm.example.com'. Every team and virtual key this definition creates lives on this gateway." +} + +variable "litellm_admin_api_key" { + type = string + sensitive = true + description = "LiteLLM admin key the building block authenticates with. It needs permission to create teams and keys." +} + +variable "litellm_platform_type_name" { + type = string + default = "LiteLLM" + description = "Name of the meshStack platform type the LiteLLM platform is registered under. It must match the platform type used by the `ai/litellm` module." +} + +variable "litellm_team_models" { + type = list(string) + default = [] + description = "Names of the models on the gateway that every team created by this definition may call. An empty list sends no allow-list to LiteLLM. Create one definition per landing zone to grant different model sets." +} + +variable "litellm_team_max_budget" { + type = number + default = 100 + description = "Spending limit of a team for one budget period, in the currency the gateway reports spend in. LiteLLM blocks the team once the limit is reached." +} + +variable "litellm_team_budget_duration" { + type = string + default = "30d" + description = "Length of one budget period, after which LiteLLM resets the spend counter. Written as a LiteLLM duration such as '30d', '7d' or '1h'." +} + +variable "meshstack" { + type = object({ + owning_workspace_identifier = string + tags = optional(map(list(string)), {}) + }) + description = "Shared meshStack context. Tags are optional and propagated to building block definition metadata." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { + git_ref = "main" + bbd_draft = true + } + description = <<-EOT + `git_ref`: Hub release reference. Set to a tag (e.g. 'v1.2.3') or branch or commit sha of meshcloud/meshstack-hub repo. + `bbd_draft`: If true, allows changing the building block definition for upgrading dependent building blocks. + EOT +} + +output "building_block_definition" { + description = "BBD is consumed in building block compositions. Add it to `spec.mandatory_building_block_refs` of an AI landing zone so meshStack provisions the team when a tenant is created." + value = { + uuid = meshstack_building_block_definition.this.metadata.uuid + version_ref = var.hub.bbd_draft ? meshstack_building_block_definition.this.version_latest : meshstack_building_block_definition.this.version_latest_release + } +} + +resource "meshstack_building_block_definition" "this" { + metadata = { + owned_by_workspace = var.meshstack.owning_workspace_identifier + tags = var.meshstack.tags + } + + spec = { + display_name = "LiteLLM Team" + symbol = "https://raw.githubusercontent.com/meshcloud/meshstack-hub/${var.hub.git_ref}/modules/ai/litellm-team/buildingblock/logo.png" + description = "Creates the LiteLLM team of a tenant with a budget, and a virtual key scoped to that team." + support_url = "https://docs.litellm.ai/docs/proxy/virtual_keys" + target_type = "TENANT_LEVEL" + run_transparency = true + supported_platforms = [{ name = var.litellm_platform_type_name }] + + # The application team never orders this block by hand. An AI landing zone lists it in + # `spec.mandatory_building_block_refs`, so meshStack provisions the team and the key when the + # tenant is created. + use_in_landing_zones_only = true + + readme = chomp(<<-EOT + This building block gives your project its own team on the LiteLLM gateway, with a budget and + a virtual key that is scoped to that team. meshStack creates both when your tenant in the AI + landing zone is created, so there is nothing to order and nothing to fill in. + + ## 🎯 When to use it + + Use this building block when you: + - Want a governed OpenAI-compatible endpoint for your application instead of a credential shared across the whole platform. + - Need spend for your project to be counted and capped on its own. + - Want the platform team to decide which models you may call, through the landing zone you picked. + + ## πŸ’‘ Usage examples + + **Example 1: A chat assistant in a web application** + A team creates a tenant in the AI landing zone and reads the base URL and the virtual key from + the outputs. The team stores both in a Kubernetes secret and points the OpenAI client library + of the application at them. + + **Example 2: Keeping an evaluation job inside a budget** + A team runs a nightly job that grades model answers. The job uses the same virtual key, so its + spend counts against the project budget and LiteLLM stops the calls before the budget is + exceeded rather than after the invoice arrives. + + ## πŸ”‘ Calling the gateway + + The `api_base` output already ends in `/v1`. Send the virtual key as a bearer token: + + ```sh + curl "$API_BASE/models" \ + -H "Authorization: Bearer $VIRTUAL_KEY" + ``` + + LiteLLM returns the virtual key once, when the building block runs, and never shows it again. + Copy it into your own secret store. + + ## πŸ“Š Shared Responsibility + + | Responsibility | Platform Team | Application Team | + |---|:---:|:---:| + | Operate the LiteLLM gateway and the model backends behind it | βœ… | ❌ | + | Set the budget, the budget period and the allowed models per landing zone | βœ… | ❌ | + | Create the team and the virtual key when the tenant is created | βœ… | ❌ | + | Store the virtual key in the application's own secret store | ❌ | βœ… | + | Stay within the granted budget and the allowed models | ❌ | βœ… | + | Build and operate the application that calls the gateway | ❌ | βœ… | + EOT + ) + } + + version_spec = { + draft = var.hub.bbd_draft + deletion_mode = "DELETE" + + # One tenant is one LiteLLM team. Applying the block a second time in the same tenant would + # create a second team and a second key, so meshStack refuses it. + only_apply_once_per_tenant = true + + implementation = { + terraform = { + terraform_version = "1.12.2" + repository_url = "https://github.com/meshcloud/meshstack-hub.git" + repository_path = "modules/ai/litellm-team/buildingblock" + ref_name = var.hub.git_ref + async = false + use_mesh_http_backend_fallback = true + } + } + + inputs = { + litellm_api_base = { + display_name = "LiteLLM API Base URL" + description = "Base URL of the LiteLLM gateway the team is created on." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.litellm_api_base) + } + + litellm_api_key = { + display_name = "LiteLLM Admin Key" + description = "Admin key the building block authenticates with against the LiteLLM API." + type = "STRING" + assignment_type = "STATIC" + sensitive = { + argument = { + secret_value = var.litellm_admin_api_key + secret_version = nonsensitive(sha256(var.litellm_admin_api_key)) + } + } + } + + workspace_identifier = { + display_name = "Workspace Identifier" + description = "Identifier of the meshStack workspace, used in the team alias and in the team metadata." + type = "STRING" + assignment_type = "WORKSPACE_IDENTIFIER" + } + + project_identifier = { + display_name = "Project Identifier" + description = "Identifier of the meshStack project, used in the team alias and in the team metadata." + type = "STRING" + assignment_type = "PROJECT_IDENTIFIER" + } + + meshstack_tenant_uuid = { + display_name = "Tenant UUID" + description = "UUID of the meshStack tenant, written to the team metadata so an operator can trace a team back to its tenant." + type = "STRING" + assignment_type = "MESHSTACK_TENANT_UUID" + } + + models = { + display_name = "Allowed Models" + description = "Names of the models on the gateway that the team may call." + type = "CODE" + assignment_type = "STATIC" + # jsonencode twice is correct, see https://registry.terraform.io/providers/meshcloud/meshstack/latest/docs/resources/building_block_definition#argument-1 + argument = jsonencode(jsonencode(var.litellm_team_models)) + } + + max_budget = { + display_name = "Budget" + description = "Spending limit of the team for one budget period." + type = "INTEGER" + assignment_type = "STATIC" + argument = jsonencode(var.litellm_team_max_budget) + } + + budget_duration = { + display_name = "Budget Duration" + description = "Length of one budget period, for example '30d'." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.litellm_team_budget_duration) + } + } + + outputs = { + team_id = { + display_name = "Team ID" + description = "ID of the LiteLLM team. It becomes the platform tenant ID, so building blocks ordered later can bind their resources to this team." + type = "STRING" + assignment_type = "PLATFORM_TENANT_ID" + } + + team_alias = { + display_name = "Team Alias" + description = "Alias of the LiteLLM team, shown in the LiteLLM UI." + type = "STRING" + assignment_type = "NONE" + } + + virtual_key = { + display_name = "Virtual Key" + description = "The key the application sends as a bearer token. LiteLLM returns it only at creation." + type = "STRING" + assignment_type = "NONE" + } + + key_id = { + display_name = "Key ID" + description = "Hash LiteLLM identifies the virtual key by." + type = "STRING" + assignment_type = "NONE" + } + + api_base = { + display_name = "API Base URL" + description = "OpenAI-compatible base URL of the gateway, including the '/v1' suffix." + type = "STRING" + assignment_type = "NONE" + } + + summary = { + display_name = "Summary" + type = "STRING" + assignment_type = "SUMMARY" + } + } + } +} + +terraform { + required_version = ">= 1.12.0" + + required_providers { + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.23.0" + } + } +} From aa6db084bc59ba9dd46028d5a67d22de775f004f Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:39:10 +0200 Subject: [PATCH 25/60] feat(ai/azure-openai): register an Azure OpenAI deployment as a LiteLLM backend Adds the Azure entry in the pluggable model layer of the ai-platform architecture, next to stackit/model-serving. The module creates a single litellm_model resource and no Azure resources, so it declares requiresBackplane: false and needs no Azure identity. The building block definition mirrors stackit/model-serving: TENANT_LEVEL and orderable, with team_id filled from PLATFORM_TENANT_ID. LiteLLM then offers the model to the ordering tenant's team alone and attributes its spend to that team. Azure OpenAI addresses a deployment rather than a model. The provider joins custom_llm_provider and base_model into 'azure/', so azure_deployment_name carries the Azure deployment name. Co-Authored-By: Claude Opus 5 (1M context) --- .../ai/azure-openai/buildingblock/README.md | 78 +++++ .../buildingblock/SUMMARY.md.tftpl | 27 ++ .../ai/azure-openai/buildingblock/logo.png | Bin 0 -> 5872 bytes modules/ai/azure-openai/buildingblock/main.tf | 24 ++ .../ai/azure-openai/buildingblock/outputs.tf | 26 ++ .../ai/azure-openai/buildingblock/provider.tf | 4 + .../azure-openai/buildingblock/variables.tf | 49 +++ .../ai/azure-openai/buildingblock/versions.tf | 15 + .../ai/azure-openai/meshstack_integration.tf | 291 ++++++++++++++++++ 9 files changed, 514 insertions(+) create mode 100644 modules/ai/azure-openai/buildingblock/README.md create mode 100644 modules/ai/azure-openai/buildingblock/SUMMARY.md.tftpl create mode 100644 modules/ai/azure-openai/buildingblock/logo.png create mode 100644 modules/ai/azure-openai/buildingblock/main.tf create mode 100644 modules/ai/azure-openai/buildingblock/outputs.tf create mode 100644 modules/ai/azure-openai/buildingblock/provider.tf create mode 100644 modules/ai/azure-openai/buildingblock/variables.tf create mode 100644 modules/ai/azure-openai/buildingblock/versions.tf create mode 100644 modules/ai/azure-openai/meshstack_integration.tf diff --git a/modules/ai/azure-openai/buildingblock/README.md b/modules/ai/azure-openai/buildingblock/README.md new file mode 100644 index 00000000..d77cfe36 --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/README.md @@ -0,0 +1,78 @@ +--- +name: Azure OpenAI Model Backend +supportedPlatforms: + - ai +description: Registers an Azure OpenAI deployment as a model backend on the LiteLLM gateway, scoped to the tenant's team. +# The module talks only to the LiteLLM admin API. It creates nothing in Azure and receives the +# Azure OpenAI endpoint, key and deployment name as static inputs, so there is no cloud-side setup +# to perform ahead of time. +requiresBackplane: false +--- + +# Azure OpenAI Model Backend Building Block + +This building block registers an existing Azure OpenAI deployment as a model entry on the LiteLLM +gateway. It is the Azure entry in the model layer of the AI platform architecture, next to +`stackit/model-serving` for STACKIT, and it is what makes the architecture's claim of two cloud +providers true. + +The module is thin on purpose. It creates one `litellm_model` resource and no Azure resources at +all, which is why it declares `requiresBackplane: false` and needs no Azure identity. The platform +team creates the Azure OpenAI resource and its deployment once, outside this module, and passes the +endpoint, the key and the deployment name in as static inputs. + +The `team_id` input scopes the entry to one LiteLLM team, so the gateway offers the model to that +tenant alone and attributes its spend to that team. In the AI landing zone the team ID comes from +the platform tenant ID that `ai/litellm-team` produces, so the application team fills in nothing. + +Azure OpenAI addresses a **deployment**, not a model. The provider joins `custom_llm_provider` and +`base_model` into `azure/`, so `azure_deployment_name` must carry the Azure deployment +name rather than the underlying model name. + +Application teams keep calling the LiteLLM gateway. The Azure credential stays on the gateway, so +the platform team can rotate it without a change on the application side. + +The `ncecere/litellm` provider is pinned to exactly `2.0.1`. This is a deliberate exception to the +hub rule that provider constraints use `>=`, and `versions.tf` explains why. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.12.0 | +| [litellm](#requirement\_litellm) | = 2.0.1 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [litellm_model.this](https://registry.terraform.io/providers/ncecere/litellm/2.0.1/docs/resources/model) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [azure\_deployment\_name](#input\_azure\_deployment\_name) | Name of the model deployment in the Azure OpenAI resource, for example 'gpt-4o'. Azure routes on the deployment name, not on the model name. | `string` | n/a | yes | +| [azure\_openai\_api\_key](#input\_azure\_openai\_api\_key) | Key of the Azure OpenAI resource. LiteLLM stores it and sends it upstream in the 'api-key' header. | `string` | n/a | yes | +| [azure\_openai\_api\_version](#input\_azure\_openai\_api\_version) | Azure OpenAI data plane API version. '2024-10-21' is the latest dated GA version of the inference API. | `string` | `"2024-10-21"` | no | +| [azure\_openai\_endpoint](#input\_azure\_openai\_endpoint) | Endpoint of the Azure OpenAI resource, for example 'https://my-aoai.openai.azure.com'. LiteLLM calls the deployment under this host. | `string` | n/a | yes | +| [litellm\_api\_base](#input\_litellm\_api\_base) | Base URL of the LiteLLM gateway, for example 'https://litellm.example.com'. The provider talks to the admin API under this URL. | `string` | n/a | yes | +| [litellm\_api\_key](#input\_litellm\_api\_key) | LiteLLM admin key the provider authenticates with. It needs permission to register models. | `string` | n/a | yes | +| [mode](#input\_mode) | What the deployment is used for. LiteLLM accepts 'chat', 'completion', 'embedding', 'audio\_speech', 'audio\_transcription', 'image\_generation', 'video\_generation', 'batch' and 'rerank'. | `string` | `"chat"` | no | +| [model\_name](#input\_model\_name) | Name the model is offered under on the LiteLLM gateway. Application teams pass this name in the 'model' field of their requests. | `string` | n/a | yes | +| [team\_id](#input\_team\_id) | ID of the LiteLLM team the model is registered for. Only that team can call the model. Leave unset to register the model for the whole gateway. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_base](#output\_api\_base) | OpenAI-compatible base URL of the LiteLLM gateway, including the '/v1' suffix. Calls to this model go here, not to the Azure endpoint. | +| [model\_id](#output\_model\_id) | ID LiteLLM gave the model entry. | +| [model\_name](#output\_model\_name) | Name to pass in the 'model' field of a request to the gateway. | +| [summary](#output\_summary) | Summary with the model name and the endpoint to call it on. | + diff --git a/modules/ai/azure-openai/buildingblock/SUMMARY.md.tftpl b/modules/ai/azure-openai/buildingblock/SUMMARY.md.tftpl new file mode 100644 index 00000000..ce6ca454 --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/SUMMARY.md.tftpl @@ -0,0 +1,27 @@ +# Azure OpenAI Model: **${model_name}** + +## Details + +| Property | Value | +|----------|-------| +| **API Base URL** | `${api_base}` | +| **Model name** | `${model_name}` | +| **Azure deployment** | `${deployment_name}` | +| **Azure endpoint** | `${azure_endpoint}` | +| **Azure API version** | `${azure_api_version}` | +| **Model ID** | `${model_id}` | + +## Calling the model + +Call the LiteLLM gateway, not the Azure endpoint. Send your own virtual key as a bearer token and +pass the model name in the `model` field: + +```sh +curl "${api_base}/chat/completions" \ + -H "Authorization: Bearer $VIRTUAL_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "${model_name}", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +The Azure credential stays on the gateway. Your application never sees it and never talks to Azure +directly, so the platform team can rotate the credential without a change on your side. diff --git a/modules/ai/azure-openai/buildingblock/logo.png b/modules/ai/azure-openai/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..45d81b5ef200c05b49bedda8ce6d7055278198d7 GIT binary patch literal 5872 zcmZ8ldpMJS*nj3>8@4%=vJ9g{sF=vvNFj14nN!HQ6LN?gvYDhJqe#dmQw(K7&hau(bvN5U@0}K567k$SFQd5gnWc9$OzK40{VzL zp9%M><#jHG7PY%pgZnkxmIaM&^SYEk z*dPr2#xWW#OVFZL`!e5!0$!Ig2&#G`hfu=S>RdUJ)qA{tkx(`V+1YEK^uI1pwpsl4 zxoDeAUgsC+b59`SkTry7HNV4i6oktCrCib&@5RzXDdUg=@{h+Mv+}9qN3y4puH_I@ zjh0`EnNyHHkax}2UrPBihZ$1}X%k0tze6C8=FLE)jy|7Jd;RlB&iCVQ7d4ueR4V2b z=~ECANEYRyS*^BZ2#-3GgRyuB;UtvK$v>HZ^f~fu8iIPfZc*{+^r<-%FCK^WlEjC8LAd|LqT zTHtaWTKhWSR(AZ&Pqo^gfJ-6FwfI=o0>&^BW%-cH`8i7C3gD0>b-x?9kaHlaOfulv zUT2!H_cL7dJG4)kTFnpPpju(m(7j>xD&;J>q-K$GQJC=ea_Qfc@;?!ZzbIyOs=c06 ztzsR`{-9R3px*GKye=^wDhcbrv*zc3KZQ030+-<4*#gw;<&0yGY-06Z$rf9MY3YkJ2*F9oUypfnXX-1S@^lVu(dU^ zv$M0h%t^l^4*H>S`HP)s_0(xs3W-_)?M|vgw%x{e??`Zc1V2+xY@CqL=rs4IYxc|=$+2psT zsZELF#+PmdJ@QghujmzLr$?E;KgQ7;{9Qs^{6qW~{>&~+o&iQ}E?pXa>G>`r&-G8O z%n2$L1e@i6N6ht=uN?M=_2|bcnNb_dDVRh69wU35r&ZR|o+&`DycEC9d*Cr@35W&d zD3_E6sxwZ$NRE*5n@Wf)vEzE&9j0?v};VR9I=ZoZrAs1BL-#l<#W zZY>UL?Add@u&H*wT2PK{#2c;2aoOh>p{C<{t>U8tQO^@8D>%!n=92g$Wd3{#?n2{v z%N2uDIva6Yq(ZE(HwBiB@wOOyrV|knQP+*J1V0ARY>1y>lY_DYZiL+9#Ove(MBKVS zNT}|uh#fvQ+7gGKcR7IQwxI`v#=l%Nys1l10>+RrvK@r=V)EQWwsOBFZL!(AgsTzL zViTeEhdt1*X*uD>ErDpp#nx`}RcO*?&hctu;F>^zuG_qghPdC<7T2y>Ni52rCG>VP zFf=-+G+O5AQ+-$*>JTRGfnI71iEf?{qJIj3yZJP_&x1^2Ws&C;jM>CnvUg;eB-nT0 z?dGFhvgYlxXkD;i2f0QJ3PpQ7ac7k?ckY2iauXZB))_>WD~Ng@oNF>d2Y!i z&RHtYIFXuDjfP2#1A9{k^17$ZSFVgB^DOQzN!7Elt{kfkO<-D^-d7%Q6FD zLzFY!P3%s>&-#%9(3buJ*UMIXWMzI#v*U32SC*FdIyCu0H z^B1i$AW57$?LzeRrTW0H!N!d4Jb`FcE(u(XZ2&vE!jv_!`508chacQ&2jsgRjL&<# zOPNYU`0<=-xI>~Zo0M?FSLVEwa&ZX6C7y%zAc=nFuZfXS77jk^N!DVDa-2L)|Yja3zuhInmPemm%N{;OD}{t;NYCdil{F%z>jTTiHg;YJ-ok?_~1a zB%Ako5WdYKHi`1<6uBYZ3gKFE%cy;Q>-W1s%2hq1<5Wf^aDyf0w|f5G#AW8j@(kKt zkOdSl*Z+B4Um5uQTV0bDkSTl^{7Gz>Acl1hAmYa!+|34uXG8*3X4ConT~Hc=*FxO{ z=$5;kL6iqGfm#T1753g#af&=W+g@NG+DR5i6oBrsS0TdY39;$m@cHu>g$&!tM?hU? zGL+KQ)qm&D7`(+9U$A|5Hw`w-$6m8~jq2gcuQ$lRacp>=(e=SnAQ8~=!#G8>X6n>l zSA|&rVWGGlPy?~saLXMAdu`$FLl6cx?7|OGvh=r49aCUzx`&%N;y9x(`VFX|?hd|u;+rE4FaC)h)ICXK-;7UYV)XC{E&u6l=BBx70|qDl0ncYAxa7LGG) z8g1c$3ONnxryg)sLPEBVJ!p@;4|`7F3>4FN$d4P1ICBGRp(?GioL5yg3e>__?E0{R z&{rf4{7WW9YPI$OwLm)id2w7~ETGwGhv zOBJNv!HMFsgr|4zf_hOOq}*5WmvXOE28l00$}NIOa%Bj&#T>dnM02!uvjWKV7-plPS&+v5haO z2AuwJz#x95X~r$40~F4u$NNOqd;f(-NVnRX3G{ zB`zf|DW_<$J2Zy_$({zO$S7E{i*m@S>xL1ye>a$}m zlF-KfyPKeNYzSQ~%*7~{6x8DJREd`&o0VWmI)AY)UE*Rccu1Ao5ALqe&0FJ_vccXT zH#RinoduTcdvFW>{sVysQ$&ScSNI5ZygwW;#opWhg*QdR(y@o^)i;T%y>aC2=GE=y zJCAeiJjr}Y)O#7U`*}}_Z{@I+pV;ifAEX9dlw9N+ADkIkw!KP>Ms~ZiP+`4h|IVt& zMWSQP(Cp>ukKjwlPuQGib_O%x)ff21SG*h&5Nq7Gm$g z;tx<=e>l(hhnSHXv|d6q#owINVudVCyt{0Lt&tu%aDlig0?lL#C!~5p&+ouGeMEMrWH4b--OVY3XfN~V_eRu>qIz!_u5t(%q!DNco#I^nu zN>PJ@em>YEw!04HgY9P6&ge>ntYX8~?gipE-ih(CP>!0%J9qyp$x|#Sdkao5RT!|G z*%7&G*zi!bKcg48#Qi>58M<4(2$M}1X9r2@W)T^%pnn@*zd7?9nh!qA4a_983tuMZ zqs|8Us(S6O*$m&l@FsfaO5(!n*TzDrJucTDh23Ngj=dXlcA>zaomM_6hZviuxuS6G zwN7~^X4o9V+To1&e<`rGsCTHHvM`U-$0|MuJr2=?Mg~ztQsk|tt_P{crDi4h)m^B+ z^&YP``3!LF?AIkg4_;>O$M>t;Bzon|;mN_L&n&l%eXEbyvRJGSo5^kn=PWt@-QMu}yAWex(PqSi2Fbtz!wy#ekN*B4 zVM}oRtC#Z+!rK|PPWU|lQK9hWHOiA?bx4UUU!h9WlnUrxamkDH%3mg@XWA)-t7< z04jE5&otEjcU9pD+*p?b)an)uphWkngtBu6?1Rd>?W6xXU-ear25oaB*EIPcG%e#1 zP(hUk$hK2%K1Q~xgqX>F~h|u%C$8o_hTF&sY!kjhau)bzAlFkvDtf?Y;f_18Urr z4z|*wQBnsfaT|MGsXtueq`JfdXI->Mg%jY^a(z((H^H4Sk~CE_jq&f&ABbu4K-OV` zWkzTv$txxmG7+0A5o23tefpLWq78aG5?@Od+185GUm`IeJ(6N!(Hi+1H9@>w?9QZ9 zo-mh0hA4V^+s%49^Q>I}M*GmP(teIdUmeqrTt4+zo)A$q7|N&Y!W9XB&(Dou4OX}L zzDmgvH~x{V&GA$U=LrU~HV-2%Au>N+ewAbfe?@XZJYWq5C*ELKRtk17%^5401!#DW zZf-td2lY=?#5l^_xk(%#jirqm_EQ|M?fZZl$GzMVS?bx{(z)4`u1Z+Y@uen z=yza-80zaT^|Vne6+{kJKR=!r`qtcM%JN}13qA|fZ2n$1GCGUd9n&nO1YY<;S&Mg#u_{+NOfaeymBf7;*^#o|AYLO1ZA0~5QLXBc5pKOidZ+c(}Eo+I1Os-_!b zS8NW5uR|m8lTqSF%0?#dS$J(w4-hW09&Nc(MP-1)nP>y}m4c72^Fd!l#2f@pdv73P zX#RsVt&=5ta^r?+4=iN0V+HPt=?+Gh5p0!?Rt_Zw0*(=cdI}93_W3NPAIqnWePNf- zWwlJeWI&=wPZ7)KfgXm5#N%xFn=r(!U?@$;zQ5xaLVPbReRJUt>Qpys&xrk9EkFL$ zQLTCk85}-6y-$JqssqgS7seQx_Xcnd8{*__%-mHtB|QTIacM$9te^aEH;-jLO6QV3nV^%$DGyreliIcfR=b%W|$ z&%UM%WtHrJn4hMlwpx}~pGY@eeQx895I+SA z7c07VF$K1^e7O5iH`sc5c3l^llmtjz+*QstpInmZRvp$lhrqxROB%<hHjUjOY2qZ8u9(GV#ntv z@yU&zEr(*UPXDM)ie2T1l9MTWK2*`Bm1dy~I%0)!@9F9Ec!44$N~(W^7MTHlt3TLi zXQj?@26vAd}fa68fieMb# zT7P_E%l}Dr&o%4dwKy4P7K?Pao_9x&xVS}o&qT6}vVib^@oc&TJ>1Gwe_npM@Antu)!m0&%FMCRj*HGZ_quo-^WM z99+Fb#J)JKUDQ;G+r)(=SnbI1JhJWPz=XF#Wnb2=&ZUXIi{h(TyL2YyL;`qRJZ{bi>*Af-7xV~~Q~@*lwM3z;1cyep()f5 literal 0 HcmV?d00001 diff --git a/modules/ai/azure-openai/buildingblock/main.tf b/modules/ai/azure-openai/buildingblock/main.tf new file mode 100644 index 00000000..260ff48f --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/main.tf @@ -0,0 +1,24 @@ +locals { + # LiteLLM's OpenAI-compatible routes live under '/v1'. The gateway also answers without the + # prefix, but OpenAI client libraries expect it, so the output carries it. + api_base = "${trimsuffix(var.litellm_api_base, "/")}/v1" +} + +resource "litellm_model" "this" { + model_name = var.model_name + custom_llm_provider = "azure" + + # The provider joins these two into '/' and sends the result to + # LiteLLM. Azure OpenAI addresses a deployment, not a model, so base_model carries the Azure + # deployment name and the request goes to 'azure/'. + base_model = var.azure_deployment_name + + model_api_base = var.azure_openai_endpoint + model_api_key = var.azure_openai_api_key + api_version = var.azure_openai_api_version + mode = var.mode + + # When a team ID is given, LiteLLM offers the model to that team alone. The gateway keeps one + # entry per tenant, and the spend of each entry is attributed to its team. + team_id = var.team_id +} diff --git a/modules/ai/azure-openai/buildingblock/outputs.tf b/modules/ai/azure-openai/buildingblock/outputs.tf new file mode 100644 index 00000000..ea500b10 --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/outputs.tf @@ -0,0 +1,26 @@ +output "model_name" { + value = litellm_model.this.model_name + description = "Name to pass in the 'model' field of a request to the gateway." +} + +output "model_id" { + value = litellm_model.this.id + description = "ID LiteLLM gave the model entry." +} + +output "api_base" { + value = local.api_base + description = "OpenAI-compatible base URL of the LiteLLM gateway, including the '/v1' suffix. Calls to this model go here, not to the Azure endpoint." +} + +output "summary" { + description = "Summary with the model name and the endpoint to call it on." + value = templatefile("${path.module}/SUMMARY.md.tftpl", { + model_name = litellm_model.this.model_name + model_id = litellm_model.this.id + api_base = local.api_base + deployment_name = var.azure_deployment_name + azure_endpoint = var.azure_openai_endpoint + azure_api_version = var.azure_openai_api_version + }) +} diff --git a/modules/ai/azure-openai/buildingblock/provider.tf b/modules/ai/azure-openai/buildingblock/provider.tf new file mode 100644 index 00000000..7f3b4858 --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/provider.tf @@ -0,0 +1,4 @@ +provider "litellm" { + api_base = var.litellm_api_base + api_key = var.litellm_api_key +} diff --git a/modules/ai/azure-openai/buildingblock/variables.tf b/modules/ai/azure-openai/buildingblock/variables.tf new file mode 100644 index 00000000..0aa6e8dd --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/variables.tf @@ -0,0 +1,49 @@ +variable "litellm_api_base" { + type = string + description = "Base URL of the LiteLLM gateway, for example 'https://litellm.example.com'. The provider talks to the admin API under this URL." +} + +variable "litellm_api_key" { + type = string + sensitive = true + description = "LiteLLM admin key the provider authenticates with. It needs permission to register models." +} + +variable "azure_openai_endpoint" { + type = string + description = "Endpoint of the Azure OpenAI resource, for example 'https://my-aoai.openai.azure.com'. LiteLLM calls the deployment under this host." +} + +variable "azure_openai_api_key" { + type = string + sensitive = true + description = "Key of the Azure OpenAI resource. LiteLLM stores it and sends it upstream in the 'api-key' header." +} + +variable "azure_openai_api_version" { + type = string + default = "2024-10-21" + description = "Azure OpenAI data plane API version. '2024-10-21' is the latest dated GA version of the inference API." +} + +variable "azure_deployment_name" { + type = string + description = "Name of the model deployment in the Azure OpenAI resource, for example 'gpt-4o'. Azure routes on the deployment name, not on the model name." +} + +variable "model_name" { + type = string + description = "Name the model is offered under on the LiteLLM gateway. Application teams pass this name in the 'model' field of their requests." +} + +variable "mode" { + type = string + default = "chat" + description = "What the deployment is used for. LiteLLM accepts 'chat', 'completion', 'embedding', 'audio_speech', 'audio_transcription', 'image_generation', 'video_generation', 'batch' and 'rerank'." +} + +variable "team_id" { + type = string + default = null + description = "ID of the LiteLLM team the model is registered for. Only that team can call the model. Leave unset to register the model for the whole gateway." +} diff --git a/modules/ai/azure-openai/buildingblock/versions.tf b/modules/ai/azure-openai/buildingblock/versions.tf new file mode 100644 index 00000000..5806ca01 --- /dev/null +++ b/modules/ai/azure-openai/buildingblock/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.12.0" + + required_providers { + litellm = { + source = "ncecere/litellm" + # Exact pin, a deliberate exception to the hub rule that provider constraints use '>='. + # ncecere/litellm is a community provider with a single maintainer, and it has changed + # resource behaviour inside a minor release before: v1.2.0 replaced the id of litellm_key + # with a hash of the key. The same pin is used by modules/ai/litellm-team, so both modules + # move to a new provider release together, after a review of the changelog. + version = "= 2.0.1" + } + } +} diff --git a/modules/ai/azure-openai/meshstack_integration.tf b/modules/ai/azure-openai/meshstack_integration.tf new file mode 100644 index 00000000..a1171ff4 --- /dev/null +++ b/modules/ai/azure-openai/meshstack_integration.tf @@ -0,0 +1,291 @@ +variable "litellm_api_base" { + type = string + description = "Base URL of the LiteLLM gateway, for example 'https://litellm.example.com'. The model is registered on this gateway." +} + +variable "litellm_admin_api_key" { + type = string + sensitive = true + description = "LiteLLM admin key the building block authenticates with. It needs permission to register models." +} + +variable "litellm_platform_type_name" { + type = string + default = "LiteLLM" + description = "Name of the meshStack platform type the LiteLLM platform is registered under. It must match the platform type used by the `ai/litellm` module." +} + +variable "litellm_model_name" { + type = string + default = "azure-gpt-4o" + description = "Name the model is offered under on the gateway. Application teams pass this name in the 'model' field of their requests." +} + +variable "litellm_model_mode" { + type = string + default = "chat" + description = "What the deployment is used for. LiteLLM accepts 'chat', 'completion', 'embedding', 'audio_speech', 'audio_transcription', 'image_generation', 'video_generation', 'batch' and 'rerank'." +} + +variable "azure_openai_endpoint" { + type = string + description = "Endpoint of the Azure OpenAI resource, for example 'https://my-aoai.openai.azure.com'." +} + +variable "azure_openai_api_key" { + type = string + sensitive = true + description = "Key of the Azure OpenAI resource. LiteLLM stores it and sends it upstream, so no application team ever sees it." +} + +variable "azure_openai_api_version" { + type = string + default = "2024-10-21" + description = "Azure OpenAI data plane API version. '2024-10-21' is the latest dated GA version of the inference API." +} + +variable "azure_openai_deployment_name" { + type = string + default = "gpt-4o" + description = "Name of the model deployment in the Azure OpenAI resource. Azure routes on the deployment name, not on the model name." +} + +variable "meshstack" { + type = object({ + owning_workspace_identifier = string + tags = optional(map(list(string)), {}) + }) + description = "Shared meshStack context. Tags are optional and propagated to building block definition metadata." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { + git_ref = "main" + bbd_draft = true + } + description = <<-EOT + `git_ref`: Hub release reference. Set to a tag (e.g. 'v1.2.3') or branch or commit sha of meshcloud/meshstack-hub repo. + `bbd_draft`: If true, allows changing the building block definition for upgrading dependent building blocks. + EOT +} + +output "building_block_definition" { + description = "BBD is consumed in building block compositions, for example by the ai-platform reference architecture." + value = { + uuid = meshstack_building_block_definition.this.metadata.uuid + version_ref = var.hub.bbd_draft ? meshstack_building_block_definition.this.version_latest : meshstack_building_block_definition.this.version_latest_release + } +} + +resource "meshstack_building_block_definition" "this" { + metadata = { + owned_by_workspace = var.meshstack.owning_workspace_identifier + tags = var.meshstack.tags + } + + spec = { + display_name = "Azure OpenAI Model Backend" + symbol = "https://raw.githubusercontent.com/meshcloud/meshstack-hub/${var.hub.git_ref}/modules/ai/azure-openai/buildingblock/logo.png" + description = "Registers an Azure OpenAI deployment as a model backend on the LiteLLM gateway, offered to the ordering team alone." + support_url = "https://learn.microsoft.com/azure/ai-foundry/openai/" + target_type = "TENANT_LEVEL" + run_transparency = true + supported_platforms = [{ name = var.litellm_platform_type_name }] + + readme = chomp(<<-EOT + This building block adds an Azure OpenAI model to your LiteLLM team. The model is registered + for your team alone, so the gateway offers it to you and counts its spend against your budget. + You keep calling the same LiteLLM endpoint and only pass a different model name. + + ## 🎯 When to use it + + Use this building block when you: + - Need a model that Azure OpenAI serves, next to or instead of the sovereign models the platform offers by default. + - Want the Azure credential to stay on the gateway rather than in your application. + - Want the calls to appear in the same budget and the same usage reports as the rest of your model traffic. + + ## πŸ’‘ Usage examples + + **Example 1: Adding GPT-4o to an existing assistant** + A team already calls the gateway through its virtual key. It orders this building block, reads + the `model_name` output and changes the `model` field of its requests to that name. Nothing + else in the application changes. + + **Example 2: Comparing two models before choosing one** + A team wants to compare a sovereign model against Azure OpenAI on its own prompts. Both models + answer on the same endpoint and the same key, so the comparison is one field in the request. + + ## πŸ”‘ Calling the model + + Send your virtual key as a bearer token and pass the model name in the `model` field: + + ```sh + curl "$API_BASE/chat/completions" \ + -H "Authorization: Bearer $VIRTUAL_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model": "$MODEL_NAME", "messages": [{"role": "user", "content": "Hello"}]}' + ``` + + ## πŸ“Š Shared Responsibility + + | Responsibility | Platform Team | Application Team | + |---|:---:|:---:| + | Create the Azure OpenAI resource and its model deployment | βœ… | ❌ | + | Provide and rotate the Azure OpenAI credential on the gateway | βœ… | ❌ | + | Choose which Azure deployment is offered and under which model name | βœ… | ❌ | + | Watch the Azure quota the deployment draws from | βœ… | ❌ | + | Pass the model name in requests to the gateway | ❌ | βœ… | + | Stay within the granted budget | ❌ | βœ… | + | Build and operate the application that calls the model | ❌ | βœ… | + EOT + ) + } + + version_spec = { + draft = var.hub.bbd_draft + deletion_mode = "DELETE" + + # A second model entry with the same name for the same team would only duplicate the first one, + # so meshStack allows one per tenant. Register a further deployment with its own definition. + only_apply_once_per_tenant = true + + implementation = { + terraform = { + terraform_version = "1.12.2" + repository_url = "https://github.com/meshcloud/meshstack-hub.git" + repository_path = "modules/ai/azure-openai/buildingblock" + ref_name = var.hub.git_ref + async = false + use_mesh_http_backend_fallback = true + } + } + + inputs = { + litellm_api_base = { + display_name = "LiteLLM API Base URL" + description = "Base URL of the LiteLLM gateway the model is registered on." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.litellm_api_base) + } + + litellm_api_key = { + display_name = "LiteLLM Admin Key" + description = "Admin key the building block authenticates with against the LiteLLM API." + type = "STRING" + assignment_type = "STATIC" + sensitive = { + argument = { + secret_value = var.litellm_admin_api_key + secret_version = nonsensitive(sha256(var.litellm_admin_api_key)) + } + } + } + + team_id = { + display_name = "LiteLLM Team ID" + description = "ID of the LiteLLM team the model is registered for. It is the platform tenant ID that the LiteLLM Team building block produces." + type = "STRING" + assignment_type = "PLATFORM_TENANT_ID" + } + + azure_openai_endpoint = { + display_name = "Azure OpenAI Endpoint" + description = "Endpoint of the Azure OpenAI resource the deployment lives in." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.azure_openai_endpoint) + } + + azure_openai_api_key = { + display_name = "Azure OpenAI Key" + description = "Key of the Azure OpenAI resource. LiteLLM stores it and sends it upstream." + type = "STRING" + assignment_type = "STATIC" + sensitive = { + argument = { + secret_value = var.azure_openai_api_key + secret_version = nonsensitive(sha256(var.azure_openai_api_key)) + } + } + } + + azure_openai_api_version = { + display_name = "Azure OpenAI API Version" + description = "Data plane API version LiteLLM calls the deployment with." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.azure_openai_api_version) + } + + azure_deployment_name = { + display_name = "Azure Deployment Name" + description = "Name of the model deployment in the Azure OpenAI resource." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.azure_openai_deployment_name) + } + + model_name = { + display_name = "Model Name" + description = "Name the model is offered under on the gateway." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.litellm_model_name) + } + + mode = { + display_name = "Mode" + description = "What the deployment is used for, for example 'chat' or 'embedding'." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.litellm_model_mode) + } + } + + outputs = { + model_name = { + display_name = "Model Name" + description = "Name to pass in the 'model' field of a request to the gateway." + type = "STRING" + assignment_type = "NONE" + } + + model_id = { + display_name = "Model ID" + description = "ID LiteLLM gave the model entry." + type = "STRING" + assignment_type = "NONE" + } + + api_base = { + display_name = "API Base URL" + description = "OpenAI-compatible base URL of the gateway, including the '/v1' suffix." + type = "STRING" + assignment_type = "NONE" + } + + summary = { + display_name = "Summary" + type = "STRING" + assignment_type = "SUMMARY" + } + } + } +} + +terraform { + required_version = ">= 1.12.0" + + required_providers { + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.23.0" + } + } +} From 53c23dc3845afe30f670e5c49d375d56bd0e56c0 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:43:21 +0200 Subject: [PATCH 26/60] fix(kubernetes/ingress)!: let the caller configure the kubernetes and helm providers The module configured both providers itself from cluster_endpoint, cluster_ca_certificate and token. That shape blocked the composition in reference-architectures/stackit-kubernetes in two ways. It could not authenticate to a freshly created SKE cluster. stackit_ske_kubeconfig issues a client certificate and a client key, never a bearer token, and no step in a composition can mint a cluster-admin token before the cluster exists. It could not be made optional. A module that carries its own provider configuration is a legacy module, and OpenTofu rejects count, for_each and depends_on on calls to it. The restriction is transitive, so a local wrapper module does not lift it, and providers = {} fails with "Cannot override provider configuration". The composition needs count to skip ingress when expose is "none". provider.tf is gone and the three credential variables with it. The caller now configures the kubernetes and the helm provider and passes both down through the providers argument. Terragrunt units that generate a provider block are unaffected, because their generated configuration already served the module. Co-Authored-By: Claude Opus 5 (1M context) --- .../ingress/buildingblock/APP_TEAM_README.md | 19 ++++++ .../ingress/buildingblock/README.md | 64 +++++++++++++++---- .../ingress/buildingblock/provider.tf | 15 ----- .../ingress/buildingblock/variables.tf | 16 ----- 4 files changed, 70 insertions(+), 44 deletions(-) delete mode 100644 modules/kubernetes/ingress/buildingblock/provider.tf diff --git a/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md b/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md index 52e148ad..7ca0955c 100644 --- a/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md +++ b/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md @@ -56,6 +56,25 @@ spec: The hostname has to resolve to the ingress load balancer before Let's Encrypt can validate it, so create the DNS record first. +## βš™οΈ What the platform team installs + +The platform team runs this module against your cluster. The module brings no cluster credentials +of its own: the caller configures the `kubernetes` and the `helm` provider and hands both to the +module. + +```hcl +module "ingress" { + source = "github.com/meshcloud/meshstack-hub//modules/kubernetes/ingress/buildingblock?ref=main" + + providers = { + kubernetes = kubernetes + helm = helm + } + + acme_email = "platform@example.com" +} +``` + ## πŸ“Š Shared Responsibility | Responsibility | Platform Team | Application Team | diff --git a/modules/kubernetes/ingress/buildingblock/README.md b/modules/kubernetes/ingress/buildingblock/README.md index 28ef1482..79c424ef 100644 --- a/modules/kubernetes/ingress/buildingblock/README.md +++ b/modules/kubernetes/ingress/buildingblock/README.md @@ -3,8 +3,8 @@ name: Kubernetes Ingress with TLS supportedPlatforms: - kubernetes description: Installs cert-manager, the HAProxy ingress controller and a Let's Encrypt ClusterIssuer so every service in the cluster can get a public HTTPS URL with a valid certificate. -# All credentials β€” cluster and DNS provider β€” arrive as inputs, so there is nothing to set up -# on the cloud side. +# The cluster credentials arrive through the providers the caller configures and the DNS provider +# credentials arrive as inputs, so there is nothing to set up on the cloud side. requiresBackplane: false --- @@ -54,23 +54,61 @@ Five foundation units carried copies of the same `certmanager.tf`, `haproxy.tf` The module defaults match the SKE copy, which is the most current one. AKS callers set `haproxy_service_annotations` and get the newer cert-manager and the retry-backoff tuning along the way. +## The caller configures the providers + +This module carries no `provider` block. The caller configures the `kubernetes` and the `helm` +provider and passes both down through the `providers` argument of the module call. Two things +follow from that. + +**Any credential works.** The module used to take a `cluster_endpoint`, a `cluster_ca_certificate` +and a bearer `token`, which ruled out every cluster that hands out a client certificate instead of +a token. STACKIT SKE is one of them: `stackit_ske_kubeconfig` issues a client certificate and a +client key, and nothing can mint a cluster-admin token before the cluster exists. A caller that +configures the provider itself picks whichever credential its cluster offers. + +**The module call accepts `count`.** A module with its own provider configuration is a legacy +module, and OpenTofu refuses `count`, `for_each` and `depends_on` on calls to it. The restriction +is transitive, so wrapping the module in a local module does not lift it. Compositions that make +ingress optional need `count`, which is only possible without a local provider configuration. + ## Notes for platform engineers - **Providers.** Only `kubernetes` and `helm`. No cloud provider ever enters this module, so it works on SKE, AKS and anything else that speaks the Kubernetes API. Cloud-specific behaviour arrives as strings, mainly through `haproxy_service_annotations`. - **Sourced, not ordered.** There is no `meshstack_integration.tf` and no `backplane/`. Foundations source `buildingblock/` from a Terragrunt unit, and reference architectures source it from their own building block. -- **Permissions.** The token in `var.token` needs cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC. +- **Permissions.** The credentials the caller puts into the two providers need cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC. - **DNS records.** Point your DNS A record at the `haproxy_lb_ip` output. Nothing can be issued or served before that record resolves. - **First apply.** HAProxy comes up before the wildcard certificate is issued. Until the secret exists, HAProxy serves its own self-signed certificate for unmatched hosts and picks the real one up as soon as cert-manager writes it. ## Usage +The caller configures both providers and passes them into the module call. + ```hcl +provider "kubernetes" { + host = module.cluster.provider_config.host + cluster_ca_certificate = module.cluster.provider_config.cluster_ca_certificate + client_certificate = module.cluster.provider_config.client_certificate + client_key = module.cluster.provider_config.client_key +} + +provider "helm" { + kubernetes = { + host = module.cluster.provider_config.host + cluster_ca_certificate = module.cluster.provider_config.cluster_ca_certificate + client_certificate = module.cluster.provider_config.client_certificate + client_key = module.cluster.provider_config.client_key + } +} + module "ingress" { + # The call may use count, because the module configures no provider of its own. + count = var.expose == "none" ? 0 : 1 source = "github.com/meshcloud/meshstack-hub//modules/kubernetes/ingress/buildingblock?ref=main" - cluster_endpoint = var.cluster_endpoint - cluster_ca_certificate = var.cluster_ca_certificate - token = var.token + providers = { + kubernetes = kubernetes + helm = helm + } acme_email = "ske@meshcloud.io" @@ -84,11 +122,14 @@ module "ingress" { } ``` +A Terragrunt unit does the same through a `generate "provider"` block that writes both provider +configurations next to the module call. + ## Requirements | Name | Version | -| ---- | ------- | +|------|---------| | [terraform](#requirement\_terraform) | >= 1.12.0 | | [helm](#requirement\_helm) | >= 3.0.0 | | [kubernetes](#requirement\_kubernetes) | >= 2.38 | @@ -100,7 +141,7 @@ No modules. ## Resources | Name | Type | -| ---- | ---- | +|------|------| | [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.haproxy](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.issuer](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | @@ -114,7 +155,7 @@ No modules. ## Inputs | Name | Description | Type | Default | Required | -| ---- | ----------- | ---- | ------- | :------: | +|------|-------------|------|---------|:--------:| | [acme\_email](#input\_acme\_email) | Contact address Let's Encrypt uses for expiry warnings and account recovery. | `string` | n/a | yes | | [acme\_private\_key\_secret\_name](#input\_acme\_private\_key\_secret\_name) | Name of the secret in which cert-manager stores the ACME account private key. | `string` | `"letsencrypt-prod-account-key"` | no | | [acme\_server](#input\_acme\_server) | ACME directory URL. Point this at https://acme-staging-v02.api.letsencrypt.org/directory while you test, because the production endpoint has strict rate limits. | `string` | `"https://acme-v02.api.letsencrypt.org/directory"` | no | @@ -122,8 +163,6 @@ No modules. | [cert\_manager\_extra\_args](#input\_cert\_manager\_extra\_args) | Extra command line arguments for the cert-manager controller. | `list(string)` | 
[
  "--certificate-request-minimum-backoff-duration=1m"
]
| no | | [cert\_manager\_namespace](#input\_cert\_manager\_namespace) | Namespace for cert-manager and, when DNS-01 runs through STACKIT, for the STACKIT cert-manager webhook. The webhook chart expects both in the same namespace. | `string` | `"cert-manager"` | no | | [cert\_manager\_version](#input\_cert\_manager\_version) | Version of the cert-manager Helm chart. See https://github.com/cert-manager/cert-manager/releases. | `string` | `"v1.20.0"` | no | -| [cluster\_ca\_certificate](#input\_cluster\_ca\_certificate) | Cluster CA certificate, base64 encoded. | `string` | n/a | yes | -| [cluster\_endpoint](#input\_cluster\_endpoint) | IP address or hostname of the cluster control plane, without the https:// scheme. | `string` | n/a | yes | | [cluster\_issuer\_name](#input\_cluster\_issuer\_name) | Name of the ClusterIssuer. Application teams reference it from the cert-manager.io/cluster-issuer annotation on their Ingress. | `string` | `"letsencrypt-prod"` | no | | [dns01](#input\_dns01) | Enables a wildcard certificate for zone\_name via DNS-01. Set exactly one provider. Null keeps HTTP-01 per-hostname issuance. | 
object({
    zone_name = string
    stackit   = optional(object({ project_id = string, service_account_key = string }))
    route53   = optional(object({ hosted_zone_id = string, access_key_id = string, secret_access_key = string, region = optional(string, "eu-central-1") }))
  })
| `null` | no | | [haproxy\_namespace](#input\_haproxy\_namespace) | Namespace for the HAProxy ingress controller. The wildcard certificate is created here as well, so its secret survives the teardown of any application namespace. | `string` | `"haproxy-ingress"` | no | @@ -135,13 +174,12 @@ No modules. | [haproxy\_version](#input\_haproxy\_version) | Version of the haproxytech/kubernetes-ingress Helm chart. See https://github.com/haproxytech/helm-charts/blob/main/kubernetes-ingress/Chart.yaml. | `string` | `"1.49.0"` | no | | [ingress\_class\_name](#input\_ingress\_class\_name) | Name of the IngressClass the controller serves. The HTTP-01 solver of the ClusterIssuer uses the same name. | `string` | `"haproxy"` | no | | [stackit\_webhook\_version](#input\_stackit\_webhook\_version) | Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook. | `string` | `"0.4.10"` | no | -| [token](#input\_token) | Token of the service account this module runs as. It needs cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC. | `string` | n/a | yes | | [wildcard\_certificate\_name](#input\_wildcard\_certificate\_name) | Name of the wildcard Certificate and of the secret it writes, both in haproxy\_namespace. Only used when dns01 is set. | `string` | `"wildcard-tls"` | no | ## Outputs | Name | Description | -| ---- | ----------- | +|------|-------------| | [cluster\_issuer\_name](#output\_cluster\_issuer\_name) | Name of the ClusterIssuer an application references from the cert-manager.io/cluster-issuer annotation on its Ingress. | | [haproxy\_lb\_ip](#output\_haproxy\_lb\_ip) | External IP of the HAProxy LoadBalancer service. Point your DNS A record here before TLS provisioning can complete. | | [haproxy\_namespace](#output\_haproxy\_namespace) | Namespace of the HAProxy ingress controller and of the wildcard certificate secret. | diff --git a/modules/kubernetes/ingress/buildingblock/provider.tf b/modules/kubernetes/ingress/buildingblock/provider.tf deleted file mode 100644 index 7002f69d..00000000 --- a/modules/kubernetes/ingress/buildingblock/provider.tf +++ /dev/null @@ -1,15 +0,0 @@ -provider "kubernetes" { - host = "https://${var.cluster_endpoint}" - cluster_ca_certificate = base64decode(var.cluster_ca_certificate) - token = var.token -} - -# The helm provider talks to the same control plane with the same credentials, so namespaces, -# Helm releases and the resources the charts render all land in one cluster without extra wiring. -provider "helm" { - kubernetes = { - host = "https://${var.cluster_endpoint}" - cluster_ca_certificate = base64decode(var.cluster_ca_certificate) - token = var.token - } -} diff --git a/modules/kubernetes/ingress/buildingblock/variables.tf b/modules/kubernetes/ingress/buildingblock/variables.tf index 2dbe5db0..0aa98f44 100644 --- a/modules/kubernetes/ingress/buildingblock/variables.tf +++ b/modules/kubernetes/ingress/buildingblock/variables.tf @@ -1,19 +1,3 @@ -variable "cluster_endpoint" { - type = string - description = "IP address or hostname of the cluster control plane, without the https:// scheme." -} - -variable "cluster_ca_certificate" { - type = string - description = "Cluster CA certificate, base64 encoded." -} - -variable "token" { - type = string - sensitive = true - description = "Token of the service account this module runs as. It needs cluster-admin rights, because cert-manager installs CRDs and cluster-scoped RBAC." -} - variable "cert_manager_version" { type = string default = "v1.20.0" From 34240c208518d1101bf660b45db883f54f2fcdba Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:44:24 +0200 Subject: [PATCH 27/60] feat(kubernetes/ingress): size every workload for a demonstration cluster This reference architecture serves demonstrations, so every workload it deploys is now sized as small as it still runs, and every value is a variable a production consumer can raise. The cert-manager chart sets no resources at all, so its controller, cainjector, webhook and startupapicheck ran unbounded. They now request 10m CPU and 32Mi to 64Mi of memory. The memory request sits above the 32Mi the chart documents as its example, because a container that gets too little memory is OOMKilled rather than slowed down. The HAProxy chart requests 250m CPU and 400Mi of memory for the controller and for its CRD Job, and sets no limit on either. The controller now requests 100m CPU and 256Mi, the Job 50m and 64Mi. The memory limit of the controller stays at 768Mi: the pod runs HAProxy next to the Go controller and the entrypoint hands HAProxy two thirds of the cgroup limit, and users of this chart version report a reload loop below a 500Mi limit (haproxytech/kubernetes-ingress#799). haproxy_replica_count drops from the chart default of 2 to 1. One replica gives no redundancy, which the variable description and both readmes state. Co-Authored-By: Claude Opus 5 (1M context) --- .../ingress/buildingblock/APP_TEAM_README.md | 6 + .../ingress/buildingblock/README.md | 38 ++++- .../kubernetes/ingress/buildingblock/main.tf | 30 ++++ .../ingress/buildingblock/variables.tf | 157 +++++++++++++++++- 4 files changed, 226 insertions(+), 5 deletions(-) diff --git a/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md b/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md index 7ca0955c..566dbc43 100644 --- a/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md +++ b/modules/kubernetes/ingress/buildingblock/APP_TEAM_README.md @@ -75,6 +75,12 @@ module "ingress" { } ``` +**The resource requests and limits of the ingress controller and of cert-manager default to +demonstration sizes.** HAProxy runs as a single replica with 100m CPU requested, which is enough +to show a service answering over HTTPS and not enough to carry production traffic. Ask your +platform team to raise `haproxy_replica_count` and the `*_resources` variables before you put a +real workload behind this ingress. + ## πŸ“Š Shared Responsibility | Responsibility | Platform Team | Application Team | diff --git a/modules/kubernetes/ingress/buildingblock/README.md b/modules/kubernetes/ingress/buildingblock/README.md index 79c424ef..c30b1e75 100644 --- a/modules/kubernetes/ingress/buildingblock/README.md +++ b/modules/kubernetes/ingress/buildingblock/README.md @@ -79,6 +79,35 @@ ingress optional need `count`, which is only possible without a local provider c - **DNS records.** Point your DNS A record at the `haproxy_lb_ip` output. Nothing can be issued or served before that record resolves. - **First apply.** HAProxy comes up before the wildcard certificate is issued. Until the secret exists, HAProxy serves its own self-signed certificate for unmatched hosts and picks the real one up as soon as cert-manager writes it. +## Resource sizing + +Every workload this module installs gets an explicit resource request and limit, and every one of +those values is a variable. **The defaults are sized for a demonstration cluster and a production +consumer has to raise them.** The defaults keep the whole ingress stack under roughly 300m CPU and +600Mi of requested memory, so it fits next to an application on a two-node cluster. + +| Workload | Request | Limit | Variable | +|---|---|---|---| +| HAProxy controller | 100m / 256Mi | 500m / 768Mi | `haproxy_resources` | +| HAProxy CRD Job | 50m / 64Mi | 200m / 256Mi | `haproxy_crdjob_resources` | +| cert-manager controller | 10m / 64Mi | 200m / 256Mi | `cert_manager_resources` | +| cert-manager cainjector | 10m / 64Mi | 200m / 256Mi | `cert_manager_cainjector_resources` | +| cert-manager webhook | 10m / 32Mi | 100m / 128Mi | `cert_manager_webhook_resources` | +| cert-manager startupapicheck | 10m / 32Mi | 100m / 128Mi | `cert_manager_startupapicheck_resources` | +| STACKIT cert-manager webhook | 10m / 64Mi | 100m / 128Mi | `stackit_webhook_resources` | + +`haproxy_replica_count` defaults to 1 for the same reason. One replica gives no redundancy: every +restart and every node drain interrupts ingress traffic. + +The HAProxy limit is the one value that cannot go much lower. The pod runs HAProxy and the Go +controller side by side, and the container entrypoint hands HAProxy two thirds of the cgroup +memory limit. Users of chart 1.49.0 report that HAProxy reloads in a loop instead of serving +traffic when the memory limit stays below 500Mi, and a maintainer recommends at least 1Gi. + +For production, raise the HAProxy controller to 500m / 1Gi requested with a 2Gi limit, run at +least two replicas on separate nodes, and give the cert-manager controller and cainjector 100m +CPU and 512Mi memory each. + ## Usage The caller configures both providers and passes them into the module call. @@ -159,20 +188,27 @@ No modules. | [acme\_email](#input\_acme\_email) | Contact address Let's Encrypt uses for expiry warnings and account recovery. | `string` | n/a | yes | | [acme\_private\_key\_secret\_name](#input\_acme\_private\_key\_secret\_name) | Name of the secret in which cert-manager stores the ACME account private key. | `string` | `"letsencrypt-prod-account-key"` | no | | [acme\_server](#input\_acme\_server) | ACME directory URL. Point this at https://acme-staging-v02.api.letsencrypt.org/directory while you test, because the production endpoint has strict rate limits. | `string` | `"https://acme-v02.api.letsencrypt.org/directory"` | no | +| [cert\_manager\_cainjector\_resources](#input\_cert\_manager\_cainjector\_resources) | Resource requests and limits of the cert-manager cainjector. The default is sized for a
demonstration cluster and a production consumer has to raise it.

The cainjector watches every Secret in the cluster, so its memory grows with the number of
Secrets. cert-manager issue #6217 reports it reaching gigabytes on large clusters. The limit
here is `256Mi` because a demonstration cluster holds few Secrets, and a production cluster
wants `512Mi` or more together with the `--namespace` flag that narrows the watch. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "200m",
    "memory": "256Mi"
  },
  "requests": {
    "cpu": "10m",
    "memory": "64Mi"
  }
}
| no | | [cert\_manager\_crds\_keep](#input\_cert\_manager\_crds\_keep) | Keep the cert-manager CRDs when the Helm release is destroyed. Keeping them preserves existing Certificate and ClusterIssuer objects across a reinstall. | `bool` | `false` | no | | [cert\_manager\_extra\_args](#input\_cert\_manager\_extra\_args) | Extra command line arguments for the cert-manager controller. | `list(string)` | 
[
  "--certificate-request-minimum-backoff-duration=1m"
]
| no | | [cert\_manager\_namespace](#input\_cert\_manager\_namespace) | Namespace for cert-manager and, when DNS-01 runs through STACKIT, for the STACKIT cert-manager webhook. The webhook chart expects both in the same namespace. | `string` | `"cert-manager"` | no | +| [cert\_manager\_resources](#input\_cert\_manager\_resources) | Resource requests and limits of the cert-manager controller. The default is sized for a
demonstration cluster and a production consumer has to raise it.

The cert-manager Helm chart sets no resources at all and documents `10m` CPU and `32Mi` memory
as its example request. The memory request here is `64Mi` instead, because the controller keeps
informer caches for Certificates, Secrets and Ingresses and a container that runs out of memory
is OOMKilled rather than slowed down. A cluster that issues certificates continuously wants
`100m` CPU and `512Mi` memory. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "200m",
    "memory": "256Mi"
  },
  "requests": {
    "cpu": "10m",
    "memory": "64Mi"
  }
}
| no | +| [cert\_manager\_startupapicheck\_resources](#input\_cert\_manager\_startupapicheck\_resources) | Resource requests and limits of the cert-manager startupapicheck Job. The default is sized for
a demonstration cluster and a production consumer has to raise it.

The Job runs once per install, checks that the webhook answers and then exits, so it never
holds resources for long. Its request still has to fit on a node, which is why it is kept this
small. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "100m",
    "memory": "128Mi"
  },
  "requests": {
    "cpu": "10m",
    "memory": "32Mi"
  }
}
| no | | [cert\_manager\_version](#input\_cert\_manager\_version) | Version of the cert-manager Helm chart. See https://github.com/cert-manager/cert-manager/releases. | `string` | `"v1.20.0"` | no | +| [cert\_manager\_webhook\_resources](#input\_cert\_manager\_webhook\_resources) | Resource requests and limits of the cert-manager admission webhook. The default is sized for a
demonstration cluster and a production consumer has to raise it.

The webhook validates cert-manager objects and holds no cache, so it is the smallest of the
three cert-manager pods. Every apply that touches a Certificate or an Issuer goes through it,
so keep the limit above the request. A production cluster wants `100m` CPU and `256Mi` memory. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "100m",
    "memory": "128Mi"
  },
  "requests": {
    "cpu": "10m",
    "memory": "32Mi"
  }
}
| no | | [cluster\_issuer\_name](#input\_cluster\_issuer\_name) | Name of the ClusterIssuer. Application teams reference it from the cert-manager.io/cluster-issuer annotation on their Ingress. | `string` | `"letsencrypt-prod"` | no | | [dns01](#input\_dns01) | Enables a wildcard certificate for zone\_name via DNS-01. Set exactly one provider. Null keeps HTTP-01 per-hostname issuance. | 
object({
    zone_name = string
    stackit   = optional(object({ project_id = string, service_account_key = string }))
    route53   = optional(object({ hosted_zone_id = string, access_key_id = string, secret_access_key = string, region = optional(string, "eu-central-1") }))
  })
| `null` | no | +| [haproxy\_crdjob\_resources](#input\_haproxy\_crdjob\_resources) | Resource requests and limits of the Job the HAProxy chart runs to install its CRDs. The default
is sized for a demonstration cluster and a production consumer has to raise it.

The chart requests `250m` CPU and `400Mi` memory for this Job. The Job applies a handful of
CRDs and exits, so a much smaller request is enough, and a smaller request also means the Job
still schedules on a small node. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "200m",
    "memory": "256Mi"
  },
  "requests": {
    "cpu": "50m",
    "memory": "64Mi"
  }
}
| no | | [haproxy\_namespace](#input\_haproxy\_namespace) | Namespace for the HAProxy ingress controller. The wildcard certificate is created here as well, so its secret survives the teardown of any application namespace. | `string` | `"haproxy-ingress"` | no | | [haproxy\_release\_name](#input\_haproxy\_release\_name) | Helm release name of the HAProxy ingress controller. The chart names the controller Service '-kubernetes-ingress'. | `string` | `"haproxy"` | no | -| [haproxy\_replica\_count](#input\_haproxy\_replica\_count) | Number of HAProxy ingress controller replicas. | `number` | `2` | no | +| [haproxy\_replica\_count](#input\_haproxy\_replica\_count) | Number of HAProxy ingress controller replicas. The default of 1 is sized for a demonstration cluster and gives no redundancy: every restart or node drain interrupts ingress traffic. Production wants at least 2, spread over separate nodes. | `number` | `1` | no | +| [haproxy\_resources](#input\_haproxy\_resources) | Resource requests and limits of the HAProxy ingress controller. The default is sized for a
demonstration cluster and a production consumer has to raise it.

The chart requests `250m` CPU and `400Mi` memory and sets no limit. The pod runs two processes,
HAProxy itself and the Go controller, and the container entrypoint hands HAProxy two thirds of
the cgroup memory limit. Users of this chart version report that HAProxy reloads in a loop
instead of serving traffic when the memory limit stays below `500Mi`, and a maintainer
recommends at least `1Gi` (haproxytech/kubernetes-ingress issue #799). The `768Mi` limit here is
the smallest value that clears that threshold with headroom. Production wants `1Gi` to `2Gi`
and a CPU limit that matches the traffic the controller has to terminate. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "500m",
    "memory": "768Mi"
  },
  "requests": {
    "cpu": "100m",
    "memory": "256Mi"
  }
}
| no | | [haproxy\_service\_annotations](#input\_haproxy\_service\_annotations) | Annotations on the HAProxy controller Service. The cloud provider reads them to configure the
load balancer. Two values matter in practice:
- AKS needs `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path = "/healthz"`.
- STACKIT uses `lb.stackit.cloud/internal-lb` to keep the load balancer off the public internet. | `map(string)` | `{}` | no | | [haproxy\_service\_type](#input\_haproxy\_service\_type) | Service type of the HAProxy ingress controller. | `string` | `"LoadBalancer"` | no | | [haproxy\_timeout](#input\_haproxy\_timeout) | Seconds to wait for the HAProxy Helm release to become ready. The default of 20 minutes covers the time a cloud provider takes to provision the load balancer. | `number` | `1200` | no | | [haproxy\_version](#input\_haproxy\_version) | Version of the haproxytech/kubernetes-ingress Helm chart. See https://github.com/haproxytech/helm-charts/blob/main/kubernetes-ingress/Chart.yaml. | `string` | `"1.49.0"` | no | | [ingress\_class\_name](#input\_ingress\_class\_name) | Name of the IngressClass the controller serves. The HTTP-01 solver of the ClusterIssuer uses the same name. | `string` | `"haproxy"` | no | +| [stackit\_webhook\_resources](#input\_stackit\_webhook\_resources) | Resource requests and limits of the STACKIT cert-manager webhook. Only used when dns01.stackit
is set. The default is sized for a demonstration cluster and a production consumer has to raise
it.

The chart sets no resources and its values file states that `100m` CPU and `128Mi` memory are
enough for the webhook, which is what the limit uses. The webhook answers one DNS-01 challenge
per certificate renewal, so the request stays well below that. Production wants the chart's own
figures as the request as well. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "100m",
    "memory": "128Mi"
  },
  "requests": {
    "cpu": "10m",
    "memory": "64Mi"
  }
}
| no | | [stackit\_webhook\_version](#input\_stackit\_webhook\_version) | Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook. | `string` | `"0.4.10"` | no | | [wildcard\_certificate\_name](#input\_wildcard\_certificate\_name) | Name of the wildcard Certificate and of the secret it writes, both in haproxy\_namespace. Only used when dns01 is set. | `string` | `"wildcard-tls"` | no | diff --git a/modules/kubernetes/ingress/buildingblock/main.tf b/modules/kubernetes/ingress/buildingblock/main.tf index 468ac221..cd30f562 100644 --- a/modules/kubernetes/ingress/buildingblock/main.tf +++ b/modules/kubernetes/ingress/buildingblock/main.tf @@ -10,6 +10,23 @@ locals { # The chart derives the controller Service name from the release name. haproxy_service_name = "${var.haproxy_release_name}-kubernetes-ingress" + + # Helm renders these values into the pod spec as YAML, and the API server rejects a resource + # quantity that is null, so drop every field the caller left unset. + resources = { + for name, spec in { + cert_manager = var.cert_manager_resources + cert_manager_webhook = var.cert_manager_webhook_resources + cert_manager_cainjector = var.cert_manager_cainjector_resources + cert_manager_startupapicheck = var.cert_manager_startupapicheck_resources + stackit_webhook = var.stackit_webhook_resources + haproxy = var.haproxy_resources + haproxy_crdjob = var.haproxy_crdjob_resources + } : name => { + requests = { for key, value in spec.requests : key => value if value != null } + limits = { for key, value in spec.limits : key => value if value != null } + } + } } resource "kubernetes_namespace_v1" "cert_manager" { @@ -36,6 +53,13 @@ resource "helm_release" "cert_manager" { keep = var.cert_manager_crds_keep } extraArgs = var.cert_manager_extra_args + + # The chart ships no resources for any of its four workloads, so each of them would run + # unbounded without these values. + resources = local.resources.cert_manager + webhook = { resources = local.resources.cert_manager_webhook } + cainjector = { resources = local.resources.cert_manager_cainjector } + startupapicheck = { resources = local.resources.cert_manager_startupapicheck } }) ] } @@ -81,6 +105,7 @@ resource "helm_release" "stackit_cert_manager_webhook" { enabled = true secretName = kubernetes_secret_v1.stackit_dns01[0].metadata[0].name } + resources = local.resources.stackit_webhook }) ] @@ -170,11 +195,16 @@ resource "helm_release" "haproxy" { values = [ yamlencode({ + # The chart requests 250m CPU and 400Mi memory for the controller and for the CRD Job, and + # sets no limit on either. + crdjob = { resources = local.resources.haproxy_crdjob } + controller = merge( { replicaCount = var.haproxy_replica_count ingressClass = var.ingress_class_name ingressClassResource = { name = var.ingress_class_name } + resources = local.resources.haproxy service = { type = var.haproxy_service_type annotations = var.haproxy_service_annotations diff --git a/modules/kubernetes/ingress/buildingblock/variables.tf b/modules/kubernetes/ingress/buildingblock/variables.tf index 0aa98f44..138805e5 100644 --- a/modules/kubernetes/ingress/buildingblock/variables.tf +++ b/modules/kubernetes/ingress/buildingblock/variables.tf @@ -24,6 +24,89 @@ variable "cert_manager_crds_keep" { description = "Keep the cert-manager CRDs when the Helm release is destroyed. Keeping them preserves existing Certificate and ClusterIssuer objects across a reinstall." } +variable "cert_manager_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "10m", memory = "64Mi" } + limits = { cpu = "200m", memory = "256Mi" } + } + description = <<-EOT + Resource requests and limits of the cert-manager controller. The default is sized for a + demonstration cluster and a production consumer has to raise it. + + The cert-manager Helm chart sets no resources at all and documents `10m` CPU and `32Mi` memory + as its example request. The memory request here is `64Mi` instead, because the controller keeps + informer caches for Certificates, Secrets and Ingresses and a container that runs out of memory + is OOMKilled rather than slowed down. A cluster that issues certificates continuously wants + `100m` CPU and `512Mi` memory. + EOT +} + +variable "cert_manager_webhook_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "10m", memory = "32Mi" } + limits = { cpu = "100m", memory = "128Mi" } + } + description = <<-EOT + Resource requests and limits of the cert-manager admission webhook. The default is sized for a + demonstration cluster and a production consumer has to raise it. + + The webhook validates cert-manager objects and holds no cache, so it is the smallest of the + three cert-manager pods. Every apply that touches a Certificate or an Issuer goes through it, + so keep the limit above the request. A production cluster wants `100m` CPU and `256Mi` memory. + EOT +} + +variable "cert_manager_cainjector_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "10m", memory = "64Mi" } + limits = { cpu = "200m", memory = "256Mi" } + } + description = <<-EOT + Resource requests and limits of the cert-manager cainjector. The default is sized for a + demonstration cluster and a production consumer has to raise it. + + The cainjector watches every Secret in the cluster, so its memory grows with the number of + Secrets. cert-manager issue #6217 reports it reaching gigabytes on large clusters. The limit + here is `256Mi` because a demonstration cluster holds few Secrets, and a production cluster + wants `512Mi` or more together with the `--namespace` flag that narrows the watch. + EOT +} + +variable "cert_manager_startupapicheck_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "10m", memory = "32Mi" } + limits = { cpu = "100m", memory = "128Mi" } + } + description = <<-EOT + Resource requests and limits of the cert-manager startupapicheck Job. The default is sized for + a demonstration cluster and a production consumer has to raise it. + + The Job runs once per install, checks that the webhook answers and then exits, so it never + holds resources for long. Its request still has to fit on a node, which is why it is kept this + small. + EOT +} + variable "haproxy_version" { type = string default = "1.49.0" @@ -44,10 +127,54 @@ variable "haproxy_release_name" { variable "haproxy_replica_count" { type = number - # Equals the chart default. It is exposed so a foundation can scale the controller without - # having to reach into the chart values. - default = 2 - description = "Number of HAProxy ingress controller replicas." + # The chart defaults to 2. One replica is enough to serve traffic and it halves what the + # controller costs on a demonstration cluster. + default = 1 + description = "Number of HAProxy ingress controller replicas. The default of 1 is sized for a demonstration cluster and gives no redundancy: every restart or node drain interrupts ingress traffic. Production wants at least 2, spread over separate nodes." +} + +variable "haproxy_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "100m", memory = "256Mi" } + limits = { cpu = "500m", memory = "768Mi" } + } + description = <<-EOT + Resource requests and limits of the HAProxy ingress controller. The default is sized for a + demonstration cluster and a production consumer has to raise it. + + The chart requests `250m` CPU and `400Mi` memory and sets no limit. The pod runs two processes, + HAProxy itself and the Go controller, and the container entrypoint hands HAProxy two thirds of + the cgroup memory limit. Users of this chart version report that HAProxy reloads in a loop + instead of serving traffic when the memory limit stays below `500Mi`, and a maintainer + recommends at least `1Gi` (haproxytech/kubernetes-ingress issue #799). The `768Mi` limit here is + the smallest value that clears that threshold with headroom. Production wants `1Gi` to `2Gi` + and a CPU limit that matches the traffic the controller has to terminate. + EOT +} + +variable "haproxy_crdjob_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "50m", memory = "64Mi" } + limits = { cpu = "200m", memory = "256Mi" } + } + description = <<-EOT + Resource requests and limits of the Job the HAProxy chart runs to install its CRDs. The default + is sized for a demonstration cluster and a production consumer has to raise it. + + The chart requests `250m` CPU and `400Mi` memory for this Job. The Job applies a handful of + CRDs and exits, so a much smaller request is enough, and a smaller request also means the Job + still schedules on a small node. + EOT } variable "haproxy_service_type" { @@ -116,6 +243,28 @@ variable "stackit_webhook_version" { description = "Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook." } +variable "stackit_webhook_resources" { + type = object({ + requests = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + limits = optional(object({ cpu = optional(string), memory = optional(string) }), {}) + }) + nullable = false + default = { + requests = { cpu = "10m", memory = "64Mi" } + limits = { cpu = "100m", memory = "128Mi" } + } + description = <<-EOT + Resource requests and limits of the STACKIT cert-manager webhook. Only used when dns01.stackit + is set. The default is sized for a demonstration cluster and a production consumer has to raise + it. + + The chart sets no resources and its values file states that `100m` CPU and `128Mi` memory are + enough for the webhook, which is what the limit uses. The webhook answers one DNS-01 challenge + per certificate renewal, so the request stays well below that. Production wants the chart's own + figures as the request as well. + EOT +} + variable "dns01" { description = "Enables a wildcard certificate for zone_name via DNS-01. Set exactly one provider. Null keeps HTTP-01 per-hostname issuance." type = object({ From 1c38225f5acdaf44d555bc73a487a69afbb6b828 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Wed, 12 Aug 2026 23:49:45 +0200 Subject: [PATCH 28/60] feat(stackit/dns): add STACKIT DNS zone building block Creates one DNS zone in a STACKIT project, the record sets inside it, and a service account key that cert-manager and ExternalDNS use to manage records at runtime. A proof-of-concept against the live API disproved the delegated-subzone-per-tenant design for free STACKIT subdomains. Creating a two-label zone under stackit.run is rejected with "subdomain should only have one level", byte-for-byte identically with an NS delegation in place and in a second project. The primary path is therefore a single zone whose names below it are record sets. Delegation stays as an optional input for customer-owned domains, where a control test showed the cross-project machinery does exist. The zone_name validation rejects the deep-name shape at plan time, the delegation input refuses free STACKIT suffixes, and preconditions enforce trailing dots on nameservers and refuse to write an NS record for a zone the module is not creating, because an orphaned delegation resolves to SERVFAIL with no error at apply time. The backplane grants dns.admin and iam.member-admin at organization scope. STACKIT predefined role for creating service accounts could not be established, so it is documented in prose and passed through additional_organization_roles rather than guessed. Co-Authored-By: Claude Opus 5 (1M context) --- modules/stackit/dns/backplane/README.md | 145 ++++++++ modules/stackit/dns/backplane/main.tf | 78 +++++ modules/stackit/dns/backplane/outputs.tf | 4 + modules/stackit/dns/backplane/variables.tf | 34 ++ modules/stackit/dns/backplane/versions.tf | 10 + modules/stackit/dns/buildingblock/README.md | 173 ++++++++++ .../dns/buildingblock/SUMMARY.md.tftpl | 45 +++ modules/stackit/dns/buildingblock/logo.png | Bin 0 -> 1878 bytes modules/stackit/dns/buildingblock/main.tf | 157 +++++++++ modules/stackit/dns/buildingblock/outputs.tf | 49 +++ modules/stackit/dns/buildingblock/provider.tf | 14 + .../stackit/dns/buildingblock/variables.tf | 164 +++++++++ modules/stackit/dns/buildingblock/versions.tf | 13 + modules/stackit/dns/meshstack_integration.tf | 316 ++++++++++++++++++ 14 files changed, 1202 insertions(+) create mode 100644 modules/stackit/dns/backplane/README.md create mode 100644 modules/stackit/dns/backplane/main.tf create mode 100644 modules/stackit/dns/backplane/outputs.tf create mode 100644 modules/stackit/dns/backplane/variables.tf create mode 100644 modules/stackit/dns/backplane/versions.tf create mode 100644 modules/stackit/dns/buildingblock/README.md create mode 100644 modules/stackit/dns/buildingblock/SUMMARY.md.tftpl create mode 100644 modules/stackit/dns/buildingblock/logo.png create mode 100644 modules/stackit/dns/buildingblock/main.tf create mode 100644 modules/stackit/dns/buildingblock/outputs.tf create mode 100644 modules/stackit/dns/buildingblock/provider.tf create mode 100644 modules/stackit/dns/buildingblock/variables.tf create mode 100644 modules/stackit/dns/buildingblock/versions.tf create mode 100644 modules/stackit/dns/meshstack_integration.tf diff --git a/modules/stackit/dns/backplane/README.md b/modules/stackit/dns/backplane/README.md new file mode 100644 index 00000000..129480dc --- /dev/null +++ b/modules/stackit/dns/backplane/README.md @@ -0,0 +1,145 @@ +# STACKIT DNS – Backplane + +This module sets up the shared backplane configuration for the STACKIT DNS building block. It +creates a dedicated service account with a Workload Identity Federation (WIF) identity provider and +grants it the roles the building block needs: + +- **`dns.admin`** β€” create and delete DNS zones and record sets. +- **`iam.member-admin`** β€” assign the `dns.admin` role to the DNS service account the building block + creates in the zone's project. + +Authentication uses WIF (OIDC token exchange) β€” no long-lived service account key is created or +stored for the building block identity itself. + +## How one identity reaches one or two STACKIT projects + +On the usual path the building block writes into a single project: the zone, its records and the +DNS service account all live there. On the delegation path, which only works for a customer-owned +domain, it also writes the NS record into the platform team's own project, which owns the parent +zone. + +It reaches both with a single identity and a single provider configuration, because STACKIT +credentials are not bound to a project. A service account belongs to the project it was created in, +but every resource carries its own `project_id`, and access is decided by the role assignments the +account holds on the target resource. So there is no second provider, no second credential, and no +credential handed from one project to the other. + +What makes it work is the scope of the role assignment. This backplane grants `dns.admin` at +**organization** scope, because the building block is `TENANT_LEVEL`: the platform team deploys the +backplane once, long before it knows which projects future orders will land in, so the role cannot +be scoped to a single project ahead of time. The same grant covers the parent zone's project on the +delegation path. + +**Every project involved must therefore live under `organization_id`.** If the platform team keeps +its parent zone in a project outside that organization, add a +`stackit_authorization_project_role_assignment` with `dns.admin` on that project for the same +service account. + +## The one permission this module does not grant + +The building block also creates a service account and a service account key, which cert-manager and +ExternalDNS authenticate with. Creating a service account is an IAM operation, and STACKIT's +predefined role for it could not be established from public documentation. This module does not +name a role it cannot verify. + +Grant that permission yourself and pass the role name through `additional_organization_roles`. To +find it, list the roles your organization offers: + +```sh +stackit curl https://authorization.api.stackit.cloud/v2/permissions +``` + +Two ways out if you would rather not widen the identity: + +- Set `dns_service_account_enabled = false` on the building block. It then creates the zone and its + records only, and the platform team supplies the DNS key by hand. +- Define a custom role that carries the service account permissions and nothing else, then list it + in `additional_organization_roles`. + +## What the DNS key can reach + +`dns.admin` is a project role. The key the building block hands to cert-manager and ExternalDNS can +therefore write every record in every zone of the project the zone lives in. Under a free STACKIT +subdomain that is unavoidable, because such a domain admits exactly one zone at exactly one label +and every cluster below it shares that zone. Give tenants a domain you own if you need a real +boundary between them. + +## Provider requirements + +`stackit_authorization_project_role_assignment`, which the building block uses, sits behind the +STACKIT provider's `iam` experiment. The building block's own `provider.tf` sets it. The root +configuration that applies **this** backplane has to set it as well, because a backplane module +carries no provider block: + +```hcl +provider "stackit" { + default_region = "eu01" + experiments = ["iam"] +} +``` + +## Prerequisites + +- A STACKIT project where the service account will be created. +- A STACKIT service account with permissions to manage service accounts and organization-level role + assignments. +- The STACKIT organization ID under which every project involved lives. +- meshStack WIF issuer and subject from `data.meshstack_integrations.integrations`. + +## Usage + +```hcl +module "dns_backplane" { + source = "./backplane" + + project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + organization_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + + # The role your organization uses to create service accounts in tenant projects. + additional_organization_roles = [] + + workload_identity_federation = { + issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer + subjects = [""] + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.11.0 | +| [stackit](#requirement\_stackit) | >= 0.110.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_authorization_organization_role_assignment.additional](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_organization_role_assignment) | resource | +| [stackit_authorization_organization_role_assignment.dns_admin](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_organization_role_assignment) | resource | +| [stackit_authorization_organization_role_assignment.member_admin](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_organization_role_assignment) | resource | +| [stackit_service_account.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account) | resource | +| [stackit_service_account_federated_identity_provider.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account_federated_identity_provider) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_organization\_roles](#input\_additional\_organization\_roles) | Extra STACKIT roles granted to the service account at organization scope. Use this for the role your organization uses to create service accounts and service account keys in tenant projects, which the building block needs for the DNS credential. See backplane/README.md. | `list(string)` | `[]` | no | +| [organization\_id](#input\_organization\_id) | STACKIT organization ID where the service account will be granted permissions to manage DNS zones and records. Every project the building block writes into must live under this organization, including the parent zone's project on the delegation path. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | STACKIT project ID where the service account will be created. | `string` | n/a | yes | +| [service\_account\_name](#input\_service\_account\_name) | Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project. | `string` | `"mesh-dns"` | no | +| [workload\_identity\_federation](#input\_workload\_identity\_federation) | WIF issuer URL and subject list for the meshStack building block identity provider. | 
object({
    issuer   = string
    subjects = list(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [service\_account\_email](#output\_service\_account\_email) | Email of the STACKIT service account used by the buildingblock provider via WIF. | + \ No newline at end of file diff --git a/modules/stackit/dns/backplane/main.tf b/modules/stackit/dns/backplane/main.tf new file mode 100644 index 00000000..7f1188ef --- /dev/null +++ b/modules/stackit/dns/backplane/main.tf @@ -0,0 +1,78 @@ +# ───────────────────────────────────────────────────────────────────────────── +# One identity, one or two STACKIT projects +# +# On the usual path the building block writes into one project: it creates the zone, its records +# and the DNS service account there. On the delegation path, which only works for a customer-owned +# domain, it also writes the NS record into the platform team's own project, which owns the parent +# zone. +# +# It reaches both with a single identity and a single provider configuration, because STACKIT +# credentials are not bound to a project. A service account belongs to the project it was created +# in, but every resource carries its own `project_id` and access is decided by the role assignments +# the account holds on the target resource. +# +# So the backplane creates one service account and grants it the roles below at organization scope. +# Organization scope is required because the building block is `TENANT_LEVEL`: the target project +# of a future order is unknown when the platform team deploys this backplane. It also happens to +# cover the parent zone's project on the delegation path, as long as that project lives under the +# same organization. +# ───────────────────────────────────────────────────────────────────────────── + +resource "stackit_service_account" "building_block" { + project_id = var.project_id + name = var.service_account_name +} + +resource "stackit_service_account_federated_identity_provider" "building_block" { + for_each = { for i, s in var.workload_identity_federation.subjects : tostring(i) => s } + + project_id = var.project_id + service_account_email = stackit_service_account.building_block.email + name = "meshstack-${each.key}" + issuer = var.workload_identity_federation.issuer + + assertions = [ + { + item = "aud" + operator = "equals" + value = "api://AzureADTokenExchange" + }, + { + item = "sub" + operator = "equals" + value = each.value + } + ] +} + +# dns.admin allows creating and deleting zones and record sets. The building block needs it in the +# project the zone lives in, and on the delegation path also in the parent zone's project. +resource "stackit_authorization_organization_role_assignment" "dns_admin" { + resource_id = var.organization_id + role = "dns.admin" + subject = stackit_service_account.building_block.email +} + +# iam.member-admin allows assigning roles. The building block uses it to give the DNS service +# account it creates the `dns.admin` role on the zone's project and nothing beyond it. +# modules/stackit/project/backplane grants the same role for the same purpose. +resource "stackit_authorization_organization_role_assignment" "member_admin" { + resource_id = var.organization_id + role = "iam.member-admin" + subject = stackit_service_account.building_block.email +} + +# ── Creating service accounts in tenant projects ───────────────────────────── +# +# The building block also creates a service account and a service account key, which cert-manager +# and ExternalDNS authenticate with. STACKIT's predefined role for creating service accounts could +# not be established from public documentation, so this module does not name one β€” see +# backplane/README.md. Put whatever role your organization uses into +# `additional_organization_roles` rather than widening the two roles above. +resource "stackit_authorization_organization_role_assignment" "additional" { + for_each = toset(var.additional_organization_roles) + + resource_id = var.organization_id + role = each.value + subject = stackit_service_account.building_block.email +} diff --git a/modules/stackit/dns/backplane/outputs.tf b/modules/stackit/dns/backplane/outputs.tf new file mode 100644 index 00000000..3fde57dd --- /dev/null +++ b/modules/stackit/dns/backplane/outputs.tf @@ -0,0 +1,4 @@ +output "service_account_email" { + value = stackit_service_account.building_block.email + description = "Email of the STACKIT service account used by the buildingblock provider via WIF." +} diff --git a/modules/stackit/dns/backplane/variables.tf b/modules/stackit/dns/backplane/variables.tf new file mode 100644 index 00000000..b0510be0 --- /dev/null +++ b/modules/stackit/dns/backplane/variables.tf @@ -0,0 +1,34 @@ +variable "project_id" { + type = string + nullable = false + description = "STACKIT project ID where the service account will be created." +} + +variable "organization_id" { + type = string + nullable = false + description = "STACKIT organization ID where the service account will be granted permissions to manage DNS zones and records. Every project the building block writes into must live under this organization, including the parent zone's project on the delegation path." +} + +variable "workload_identity_federation" { + type = object({ + issuer = string + subjects = list(string) + }) + nullable = false + description = "WIF issuer URL and subject list for the meshStack building block identity provider." +} + +variable "service_account_name" { + type = string + default = "mesh-dns" + nullable = false + description = "Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project." +} + +variable "additional_organization_roles" { + type = list(string) + default = [] + nullable = false + description = "Extra STACKIT roles granted to the service account at organization scope. Use this for the role your organization uses to create service accounts and service account keys in tenant projects, which the building block needs for the DNS credential. See backplane/README.md." +} diff --git a/modules/stackit/dns/backplane/versions.tf b/modules/stackit/dns/backplane/versions.tf new file mode 100644 index 00000000..5c2057d3 --- /dev/null +++ b/modules/stackit/dns/backplane/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.11.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.110.0" + } + } +} diff --git a/modules/stackit/dns/buildingblock/README.md b/modules/stackit/dns/buildingblock/README.md new file mode 100644 index 00000000..be1ff347 --- /dev/null +++ b/modules/stackit/dns/buildingblock/README.md @@ -0,0 +1,173 @@ +--- +name: STACKIT DNS Zone +supportedPlatforms: + - stackit +description: Creates a STACKIT DNS zone with its record sets and a service account key that lets cert-manager and ExternalDNS manage records at runtime. +--- + +# STACKIT DNS Zone Building Block + +This building block module creates one DNS zone in a STACKIT project, the record sets inside it, and +a service account key that lets a workload manage those records at runtime. The cert-manager DNS-01 +solver and ExternalDNS both take that key. + +The module is used in two ways. A reference architecture sources it directly as a Terraform module, +which keeps STACKIT resources out of the architecture itself. A team orders it as a building block +through meshStack, wired up by the `meshstack_integration.tf` at the module root. + +## One zone, one project, one credential + +STACKIT DNS is project-scoped end to end. `stackit_dns_record_set` carries its own `project_id` and +that project must own the `zone_id`, and the SKE `extensions.dns` block has no field for a foreign +project or a foreign credential. The zone, its records and the credential that writes them +therefore all belong to one project. + +## A free STACKIT subdomain admits exactly one label + +This is measured against the live API, not assumed. Creating a two-label zone under `stackit.run` +is rejected before any delegation or project logic is reached: + +```console +$ stackit dns zone create --project-id

--name aipoc-c6a37f-sub \ + --dns-name sub.aipoc-c6a37f.stackit.run +Error: create DNS zone: 400 Bad Request, status code 400, Body: +{"message":"zone dns name sub.aipoc-c6a37f.stackit.run has one error", + "error":"subdomain 'sub.aipoc-c6a37f' should only have one level"} +``` + +The error is byte-for-byte identical with a correct NS delegation already in place in the parent +zone, and identical again in a freshly created second project. The project makes no difference. +`var.zone_name` carries a validation that rejects this shape at plan time rather than at apply. + +The same name under a customer-owned domain fails differently β€” *"collides with a parent zone in a +different project and has no delegation"* β€” so STACKIT's cross-project delegation does exist and is +project-aware. It simply never gets reached for `stackit.run`, because the name check fires first. + +So `likvid.stackit.run` is a zone, and everything below it is a record set in that zone. A cluster +reachable at `cluster1.likvid.stackit.run` with a wildcard below it needs two entries: + +```hcl +records = { + "cluster1" = { type = "A", records = ["203.0.113.17"] } + "*.cluster1" = { type = "A", records = ["203.0.113.17"] } +} +``` + +## The permission trade-off you are accepting + +With a free STACKIT subdomain, every cluster and every application under `likvid.stackit.run` lives +in one zone, in one project, written with one credential. The key this module returns therefore +**can write any record in that zone, including over a record that belongs to another cluster.** +`dns.admin` is a project role β€” it cannot be narrowed to one zone, let alone to one name. + +That is a real trade-off, not a detail: + +- Hand the key only to workloads you would trust with the whole domain. +- Give each cluster its own name prefix and treat the prefixes as a convention, not as a boundary. +- If you need a real boundary, use a domain you own and give each tenant a delegated zone of its + own in its own project. See the next section. + +## Delegation β€” for customer-owned domains only + +`var.delegation` writes the `NS` record that delegates this zone from a parent zone in another +STACKIT project. It is unset by default. Under a free STACKIT suffix the zone itself cannot exist, +so the module refuses that combination outright. + +```hcl +zone_name = "cluster1.example.com" + +delegation = { + parent_zone_project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" # the platform team's project + parent_zone_name = "example.com" +} +``` + +Two traps the code guards against, both seen in the proof-of-concept: + +- **Trailing dots are load-bearing.** STACKIT relativises a record value that does not end in a dot + against the zone, so `ns1.stackit.cloud` is stored as `ns1.stackit.cloud..`. The default + nameservers are `ns1.stackit.cloud.` and `ns2.stackit.zone.`, and a precondition rejects values + without the dot. The same applies to `CNAME`, `MX` and `NS` values in `records`. +- **An orphaned NS record fails silently.** If the delegation is published and the zone behind it + never appears, a direct query returns NOERROR with an empty answer and a referral, and a + recursive resolver returns SERVFAIL β€” with no error at apply time. The module therefore creates + the zone before the record, and a precondition refuses to delegate a name it is not creating. + +## The DNS credential + +`stackit_service_account_key.dns.json` is the raw service account key JSON that the +[STACKIT cert-manager webhook](https://github.com/stackitcloud/stackit-cert-manager-webhook) expects +as `sa.json`. Pass it, together with `zone_project_id`, to `modules/kubernetes/ingress`: + +```hcl +dns01 = { + zone_name = module.dns.zone_name + stackit = { + project_id = module.dns.zone_project_id + service_account_key = module.dns.dns_service_account_key + } +} +``` + +Set `dns_service_account_enabled = false` when records are managed through Terraform only, or when +the backplane identity may not create service accounts. Both `dns_service_account_*` outputs are +`null` in that case. + +`dns_service_account_key_ttl_days` is unset by default, so the key stays valid until it is deleted. +A key that expires has to be rotated by re-applying the building block before a certificate can +renew. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.11.0 | +| [stackit](#requirement\_stackit) | >= 0.110.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_authorization_project_role_assignment.dns](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_project_role_assignment) | resource | +| [stackit_dns_record_set.delegation](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/dns_record_set) | resource | +| [stackit_dns_record_set.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/dns_record_set) | resource | +| [stackit_dns_zone.this](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/dns_zone) | resource | +| [stackit_service_account.dns](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account) | resource | +| [stackit_service_account_key.dns](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account_key) | resource | +| [stackit_dns_zone.parent](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/data-sources/dns_zone) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [contact\_email](#input\_contact\_email) | Contact address stored on the zone. Leave empty to let STACKIT pick its own default. | `string` | `""` | no | +| [delegation](#input\_delegation) | Write the `NS` record that delegates this zone from a parent zone in another STACKIT project.
Leave unset, which is the default and the usual case.

**This works only for a domain the customer owns.** Under a free STACKIT suffix such as
`stackit.run` the zone itself cannot be created, so the delegation has nothing to point at β€” see
the API error quoted in main.tf. The module refuses that combination.

`nameservers` must carry trailing dots. STACKIT relativises a value without one against the zone,
so `ns1.stackit.cloud` is stored as `ns1.stackit.cloud..` and the delegation points nowhere. | 

object({
    parent_zone_project_id = string
    parent_zone_name       = string
    nameservers            = optional(list(string), ["ns1.stackit.cloud.", "ns2.stackit.zone."])
    ttl                    = optional(number, 3600)
  })
| `null` | no | +| [dns\_service\_account\_enabled](#input\_dns\_service\_account\_enabled) | Create a service account with `dns.admin` on the zone's project and a key for it. cert-manager's DNS-01 solver and ExternalDNS both authenticate with that key. Turn it off when the consumer manages records through Terraform only, or when the backplane identity may not create service accounts. | `bool` | `true` | no | +| [dns\_service\_account\_key\_ttl\_days](#input\_dns\_service\_account\_key\_ttl\_days) | Validity of the DNS service account key in days. Leave unset to create a key that stays valid until it is deleted. A key that expires has to be rotated by re-applying the building block before a certificate can renew. | `number` | `null` | no | +| [dns\_service\_account\_name](#input\_dns\_service\_account\_name) | Name of the DNS service account. Defaults to `mesh-dns-`, which keeps two zones in the same project apart. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | STACKIT project ID the zone, its records and the DNS service account are created in. | `string` | n/a | yes | +| [records](#input\_records) | Record sets to create in the zone, keyed by the name relative to the zone. A key of `cluster1` in
the zone `likvid.stackit.run` gives `cluster1.likvid.stackit.run`, and `*.cluster1` gives the
wildcard below it.

A value that is itself a domain name β€” the target of a `CNAME`, `MX` or `NS` record β€” must end in
a dot. STACKIT relativises a value without one against the zone, so `example.com` is stored as
`example.com.likvid.stackit.run.`.
hcl
records = {
  "cluster1"   = { type = "A", records = ["203.0.113.17"] }
  "*.cluster1" = { type = "A", records = ["203.0.113.17"] }
}
| 
map(object({
    type    = string
    records = list(string)
    ttl     = optional(number)
    comment = optional(string)
  }))
| `{}` | no | +| [service\_account\_email](#input\_service\_account\_email) | Email of the STACKIT service account the provider authenticates as via workload identity federation. Leave unset when the caller supplies its own provider configuration. | `string` | `null` | no | +| [stackit\_region](#input\_stackit\_region) | STACKIT region used as the provider's default. STACKIT DNS itself is global. Ignored when the caller supplies its own provider configuration. | `string` | `"eu01"` | no | +| [zone\_default\_ttl](#input\_zone\_default\_ttl) | Default time to live of records in the zone, in seconds. ExternalDNS and cert-manager both write records here, so a short value keeps changes visible quickly. | `number` | `300` | no | +| [zone\_description](#input\_zone\_description) | Description stored on the zone. Leave empty to store none. | `string` | `""` | no | +| [zone\_name](#input\_zone\_name) | DNS name of the zone, for example `likvid.stackit.run` or `platform.example.com`. No trailing dot.

A free STACKIT subdomain admits exactly one label. `likvid.stackit.run` is accepted and
`cluster1.likvid.stackit.run` is rejected by the API, so everything below the zone has to be a
record set in `records` rather than a zone of its own. See main.tf for the API error. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [delegation\_record\_fqdn](#output\_delegation\_record\_fqdn) | Fully qualified name of the NS record created in the parent zone. Null when `delegation` is unset, which is the usual case. | +| [dns\_service\_account\_email](#output\_dns\_service\_account\_email) | Email of the service account that manages records in the zone. Null when `dns_service_account_enabled` is false. | +| [dns\_service\_account\_key](#output\_dns\_service\_account\_key) | STACKIT service account key as raw JSON, for the cert-manager DNS-01 solver and for ExternalDNS. It holds `dns.admin` on the zone's project, so it can write every record in every zone of that project. Null when `dns_service_account_enabled` is false. | +| [record\_fqdns](#output\_record\_fqdns) | Fully qualified name of every record set the module created, keyed the same way as the `records` input. | +| [summary](#output\_summary) | Summary with the zone, its records and the DNS credential. | +| [zone\_id](#output\_zone\_id) | UUID of the zone. | +| [zone\_name](#output\_zone\_name) | DNS name of the zone, for example `likvid.stackit.run`. Use it as the ExternalDNS zone filter and as the zone the cert-manager DNS-01 solver operates on. | +| [zone\_project\_id](#output\_zone\_project\_id) | STACKIT project ID the zone lives in. The DNS-01 solver has to name the same project, because STACKIT DNS is project-scoped. | + diff --git a/modules/stackit/dns/buildingblock/SUMMARY.md.tftpl b/modules/stackit/dns/buildingblock/SUMMARY.md.tftpl new file mode 100644 index 00000000..fcf0529d --- /dev/null +++ b/modules/stackit/dns/buildingblock/SUMMARY.md.tftpl @@ -0,0 +1,45 @@ +# DNS zone: **${zone_name}** + +## Details + +| Property | Value | +|----------|-------| +| **Zone name** | `${zone_name}` | +| **Zone ID** | `${zone_id}` | +| **STACKIT project** | `${project_id}` | +| **Nameservers** | `ns1.stackit.cloud`, `ns2.stackit.zone` | + +## Records + +%{ if length(record_fqdns) == 0 ~} +The zone carries no records yet. +%{ else ~} +%{ for fqdn in record_fqdns ~} +- `${fqdn}` +%{ endfor ~} +%{ endif ~} + +Every name below the zone is a record in this zone. A free STACKIT subdomain admits no subzone, so +`app.${zone_name}` is a record set here and not a zone of its own. + +%{ if credential_shared ~} +## DNS credential + +The service account below holds the `dns.admin` role on this STACKIT project. cert-manager solves +the ACME DNS-01 challenge with its key, and ExternalDNS writes your records with it. + +| Property | Value | +|----------|-------| +| **Service account** | `${dns_email}` | +| **STACKIT project** | `${project_id}` | + +The role is project-wide, so this key can write **every record in every zone of this project**, +including records that belong to someone else. Keep it secret and hand it only to the workloads +that need it. + +Key (`sa.json` for the STACKIT cert-manager webhook): + +```json +${dns_key} +``` +%{ endif ~} diff --git a/modules/stackit/dns/buildingblock/logo.png b/modules/stackit/dns/buildingblock/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..225391782a085772b79e9ce3d491e62e933a940b GIT binary patch literal 1878 zcmZ8i2~?6<6#mQHCCjnK(l(cza%3*YfuRsEa0DL$4O$-xtp}b4MMWVo zfeMjAp^y-y03^Mv1$zUw z4poaVh9R*ztJC4?ulWj;;bZ0`Y)R7wF>gasHDB$$xQupA>x&glpWun7U!X z6yQ-nUiT1VI|15|w@CB~(tsaHH7}U30rKe!DYTvXWx~+sH{JTfGnX5D{Cm<)ZwzVm z26H|C-IKMKgpD=n^)GKu?72mac~WN-#!cL^CfN1oUA{|~ESk$PnQdWa!`#fD<*_~1 z;aOXD_4(g#yq0BUJiPGWeNNtJ$?>Kl(bf4s1{zPR>_|N_eSv2ZAv!MJ*)eL0U2-HZ z!g7_w>Li>k+$@!KgpE?hYsxj z<92SQebBlJK|$~0{N5)kPC(G4$#|bN8+WwzswtEWYb^Dj&vlsKow3)Xk$S+OZX$Jx zsZoKzX!<0dT5q@EXZ-%^k!So)>1bTu@R?<7QDe`XZExtwk9^tfx!YIzY!4ZFbstJw zImi!q?+fSOXT={=QI0D-HN1O^EwtwD*ac^5^IR$%OX5a8aJ9}BlAB)e96#2GbycBh zK+h@er#Q#-Y9Dx{q^i*W_3%9Vp-%!ePFTy>GEgYN_sBFu0a?8`5ksGGm$Vx`=%v{YowgOH%u7%C?e2AtBelE91ma+(vh#i~)A{CG zvN*-n7%P#g?+9ygkDrHRIEdKyyZ1Y*Qi|2H>sfADs*X%WB&J+zYoU#H$70Xc!UK&F z2J$50F;zliYql+K$wpXOv_tr`eqr5`DsFJ3Wy8RqXD;s)QtDls@0@M8V^tRaywr`n zO6KG^0VliqxYaQ>@5QOX>1A%6rK{2OHvKZ`FUs=hql1h?uE{o51{E5(^*oW9>bl~G)8J69EY7}oU3@HN8@=x^hCf3R zYPZM?mRCoYGUL6o<$4Hv_9jM)2~QkTQw;Nz9P6PZ-ZHl3A`C3gEoR31Quz92G8I=9 zr@XCbc`IN7pu`$swO;rQl&z0CO58kO5{5$>+RK>new09*Y0y3u*O8-C+1|CZ^$)3Z zdHOQKRGszL@*cu+ufB$mh{`?&VuEmqva1%ni&s}yaSI{<&rF@ zpIo7N+tl5oBJ-zpBz?8oD*xc;%@IF)aP8{_muwlkU&ax$DqSc(?0z{%Y*tqumQV?Q zF2GXYH0i>ad*@v!f!-^k4iHmgX$A$ABX?wCC+B5A&2q zf~v9-C8muV?wkZNhQ{r7U+b*Dgy7xHHAi#bMAaFH5#Pl1ls`I1qPza&yut}6k*-ZB zWtDvvHK~iJb`_=SB4q*zW{YW4x61YRYodHB&JjStC3PR5-ub`Y^Tq0-j{$_MVw#8m zBR&l7j}mptdjT|#V9^uR5o#oQ@f=t~|Nuh`oom fvBSwdOZPyP@64?#ZB$msKSI1O!RLl|#NK}ZQo#tq literal 0 HcmV?d00001 diff --git a/modules/stackit/dns/buildingblock/main.tf b/modules/stackit/dns/buildingblock/main.tf new file mode 100644 index 00000000..74208e06 --- /dev/null +++ b/modules/stackit/dns/buildingblock/main.tf @@ -0,0 +1,157 @@ +# ───────────────────────────────────────────────────────────────────────────── +# STACKIT DNS zone +# +# The module creates one DNS zone in a STACKIT project, the record sets inside it, and a service +# account key that lets a workload manage those records at runtime β€” the cert-manager DNS-01 solver +# and ExternalDNS both take that key. +# +# STACKIT DNS is project-scoped end to end: `stackit_dns_record_set` carries its own `project_id` +# and that project must own the `zone_id`, and the SKE `extensions.dns` block has no field for a +# foreign project or a foreign credential. So the zone, its records and the credential that writes +# them all belong to one project. +# +# ── VERIFIED: no subzones under stackit.run ────────────────────────────────── +# +# A free STACKIT subdomain gives you exactly one zone at exactly one label. A two-label zone under +# `stackit.run` is rejected by the API before any delegation or project logic is reached: +# +# stackit dns zone create --project-id

--name aipoc-c6a37f-sub \ +# --dns-name sub.aipoc-c6a37f.stackit.run +# Error: create DNS zone: 400 Bad Request, status code 400, Body: +# {"message":"zone dns name sub.aipoc-c6a37f.stackit.run has one error", +# "error":"subdomain 'sub.aipoc-c6a37f' should only have one level"} +# +# The error is byte-for-byte identical with a correct NS delegation already in place in the parent +# zone, and identical again in a freshly created second project. The project makes no difference. +# `var.zone_name` carries a validation that rejects this shape at plan time rather than at apply. +# +# The same name under a customer-owned domain fails differently β€” "collides with a parent zone in a +# different project and has no delegation" β€” so the cross-project delegation machinery does exist +# and is project-aware. It simply never gets reached for `stackit.run`, because the name check fires +# first. That is why `var.delegation` is available for customer-owned domains and refuses free +# STACKIT suffixes. +# +# ── Consequence: one zone is shared ────────────────────────────────────────── +# +# With a free STACKIT subdomain, every cluster and every application under `likvid.stackit.run` is +# a record set inside that single zone, in a single project, written with a single credential. The +# key this module returns can therefore write any record in the zone, including over a record that +# belongs to somebody else. Per-zone isolation needs a customer-owned domain. See README.md. +# ───────────────────────────────────────────────────────────────────────────── + +locals { + # Label(s) the zone sits at below the parent zone. Only used on the delegation path. + delegation_label = var.delegation == null ? null : trimsuffix(var.zone_name, ".${var.delegation.parent_zone_name}") + + dns_service_account_name = coalesce( + var.dns_service_account_name, + "mesh-dns-${replace(var.zone_name, ".", "-")}" + ) +} + +resource "stackit_dns_zone" "this" { + project_id = var.project_id + name = var.zone_name + dns_name = var.zone_name + type = "primary" + default_ttl = var.zone_default_ttl + contact_email = var.contact_email != "" ? var.contact_email : null + description = var.zone_description != "" ? var.zone_description : null +} + +# Every name below the zone is a record set here, because a free STACKIT subdomain admits no +# subzone. A cluster reachable at `cluster1.likvid.stackit.run` with a wildcard below it needs two +# entries: `cluster1` and `*.cluster1`. +resource "stackit_dns_record_set" "this" { + for_each = var.records + + project_id = var.project_id + zone_id = stackit_dns_zone.this.zone_id + name = each.key + type = each.value.type + ttl = each.value.ttl + records = each.value.records + comment = each.value.comment +} + +# ── Delegation β€” customer-owned domains only ───────────────────────────────── +# +# The NS record goes into the parent zone, which lives in the platform team's own project, while +# the zone above lives in `var.project_id`. One identity writes into both, because STACKIT +# credentials identify a service account rather than a project and every resource names its own +# `project_id`. The backplane grants that identity `dns.admin` at organization scope. +# +# Two traps this code guards against, both seen in the proof-of-concept: +# +# - Record values are relativised against the zone unless they end in a dot. `ns1.stackit.cloud` +# was stored as `ns1.stackit.cloud.aipoc-c6a37f.stackit.run.`, so the default carries the +# trailing dot and `var.delegation.nameservers` is documented to require it. +# - An orphaned NS record fails silently. The delegation is accepted and published, a direct +# query returns NOERROR with an empty answer and a referral, and a recursive resolver returns +# SERVFAIL. Nothing errors at apply time. `depends_on` therefore creates the zone first, and +# the precondition below refuses to write a record that points at a name this module is not +# creating. + +data "stackit_dns_zone" "parent" { + count = var.delegation == null ? 0 : 1 + + project_id = var.delegation.parent_zone_project_id + dns_name = var.delegation.parent_zone_name +} + +resource "stackit_dns_record_set" "delegation" { + count = var.delegation == null ? 0 : 1 + + project_id = var.delegation.parent_zone_project_id + zone_id = data.stackit_dns_zone.parent[0].zone_id + name = local.delegation_label + type = "NS" + ttl = var.delegation.ttl + records = var.delegation.nameservers + comment = "Delegates ${var.zone_name} to the STACKIT project that owns it." + + depends_on = [stackit_dns_zone.this] + + lifecycle { + precondition { + condition = endswith(var.zone_name, ".${var.delegation.parent_zone_name}") + error_message = "zone_name (${var.zone_name}) must sit below delegation.parent_zone_name (${var.delegation.parent_zone_name}), otherwise the NS record delegates a name this module does not create and the name resolves to SERVFAIL with no error at apply time." + } + + precondition { + condition = alltrue([for ns in var.delegation.nameservers : endswith(ns, ".")]) + error_message = "Every entry of delegation.nameservers must end in a dot. STACKIT relativises a value without one against the zone, so ns1.stackit.cloud is stored as ns1.stackit.cloud.. and the delegation points nowhere." + } + } +} + +# ── DNS credential ─────────────────────────────────────────────────────────── +# +# A service account holding `dns.admin` on the zone's project. cert-manager solves the ACME DNS-01 +# challenge with its key, and ExternalDNS writes the workload's records with it. +# +# `dns.admin` is a project role, so the key reaches every zone in `var.project_id` and every record +# in them. It cannot be narrowed to a single zone or a single name. + +resource "stackit_service_account" "dns" { + count = var.dns_service_account_enabled ? 1 : 0 + + project_id = var.project_id + name = local.dns_service_account_name +} + +resource "stackit_authorization_project_role_assignment" "dns" { + count = var.dns_service_account_enabled ? 1 : 0 + + resource_id = var.project_id + role = "dns.admin" + subject = stackit_service_account.dns[0].email +} + +resource "stackit_service_account_key" "dns" { + count = var.dns_service_account_enabled ? 1 : 0 + + project_id = var.project_id + service_account_email = stackit_service_account.dns[0].email + ttl_days = var.dns_service_account_key_ttl_days +} diff --git a/modules/stackit/dns/buildingblock/outputs.tf b/modules/stackit/dns/buildingblock/outputs.tf new file mode 100644 index 00000000..4f32b171 --- /dev/null +++ b/modules/stackit/dns/buildingblock/outputs.tf @@ -0,0 +1,49 @@ +output "zone_name" { + value = stackit_dns_zone.this.dns_name + description = "DNS name of the zone, for example `likvid.stackit.run`. Use it as the ExternalDNS zone filter and as the zone the cert-manager DNS-01 solver operates on." +} + +output "zone_id" { + value = stackit_dns_zone.this.zone_id + description = "UUID of the zone." +} + +output "zone_project_id" { + value = var.project_id + description = "STACKIT project ID the zone lives in. The DNS-01 solver has to name the same project, because STACKIT DNS is project-scoped." +} + +output "record_fqdns" { + value = { for name, record in stackit_dns_record_set.this : name => record.fqdn } + description = "Fully qualified name of every record set the module created, keyed the same way as the `records` input." +} + +output "delegation_record_fqdn" { + value = one(stackit_dns_record_set.delegation[*].fqdn) + description = "Fully qualified name of the NS record created in the parent zone. Null when `delegation` is unset, which is the usual case." +} + +output "dns_service_account_email" { + value = one(stackit_service_account.dns[*].email) + description = "Email of the service account that manages records in the zone. Null when `dns_service_account_enabled` is false." +} + +output "dns_service_account_key" { + value = one(stackit_service_account_key.dns[*].json) + description = "STACKIT service account key as raw JSON, for the cert-manager DNS-01 solver and for ExternalDNS. It holds `dns.admin` on the zone's project, so it can write every record in every zone of that project. Null when `dns_service_account_enabled` is false." + sensitive = true +} + +output "summary" { + description = "Summary with the zone, its records and the DNS credential." + sensitive = true + value = templatefile("${path.module}/SUMMARY.md.tftpl", { + zone_name = stackit_dns_zone.this.dns_name + zone_id = stackit_dns_zone.this.zone_id + project_id = var.project_id + record_fqdns = sort([for record in stackit_dns_record_set.this : record.fqdn]) + credential_shared = var.dns_service_account_enabled + dns_email = coalesce(one(stackit_service_account.dns[*].email), "-") + dns_key = coalesce(one(stackit_service_account_key.dns[*].json), "-") + }) +} diff --git a/modules/stackit/dns/buildingblock/provider.tf b/modules/stackit/dns/buildingblock/provider.tf new file mode 100644 index 00000000..18c2a8dd --- /dev/null +++ b/modules/stackit/dns/buildingblock/provider.tf @@ -0,0 +1,14 @@ +# One provider configuration reaches both projects. STACKIT credentials identify a service +# account, not a project, and every resource names its own `project_id`. +# +# Callers that drive this module from their own root configuration usually replace this file with a +# generated `provider.tf`. In that case `service_account_email` and `stackit_region` stay unset and +# the generated block carries the credentials instead β€” it then has to set `experiments` too. +provider "stackit" { + default_region = var.stackit_region + service_account_email = var.service_account_email + use_oidc = true + + # stackit_authorization_project_role_assignment sits behind the provider's `iam` experiment. + experiments = ["iam"] +} diff --git a/modules/stackit/dns/buildingblock/variables.tf b/modules/stackit/dns/buildingblock/variables.tf new file mode 100644 index 00000000..c5b776bb --- /dev/null +++ b/modules/stackit/dns/buildingblock/variables.tf @@ -0,0 +1,164 @@ +# ── Backplane inputs (static, set once per building block definition) ────────── + +variable "project_id" { + type = string + nullable = false + description = "STACKIT project ID the zone, its records and the DNS service account are created in." +} + +variable "service_account_email" { + type = string + nullable = true + default = null + description = "Email of the STACKIT service account the provider authenticates as via workload identity federation. Leave unset when the caller supplies its own provider configuration." +} + +variable "stackit_region" { + type = string + nullable = false + default = "eu01" + description = "STACKIT region used as the provider's default. STACKIT DNS itself is global. Ignored when the caller supplies its own provider configuration." +} + +# ── Zone ─────────────────────────────────────────────────────────────────────── + +variable "zone_name" { + type = string + nullable = false + description = <<-EOT + DNS name of the zone, for example `likvid.stackit.run` or `platform.example.com`. No trailing dot. + + A free STACKIT subdomain admits exactly one label. `likvid.stackit.run` is accepted and + `cluster1.likvid.stackit.run` is rejected by the API, so everything below the zone has to be a + record set in `records` rather than a zone of its own. See main.tf for the API error. + EOT + + validation { + condition = can(regex("^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$", var.zone_name)) + error_message = "zone_name must be a lowercase domain name with at least two labels and no trailing dot, for example likvid.stackit.run." + } + + validation { + condition = !endswith(var.zone_name, ".stackit.run") || length(split(".", var.zone_name)) == 3 + error_message = "A zone under stackit.run may carry exactly one label, for example likvid.stackit.run. STACKIT rejects a deeper name with \"subdomain '' should only have one level\". Put the deeper names into `records` instead, or use a domain you own." + } +} + +variable "zone_default_ttl" { + type = number + nullable = false + default = 300 + description = "Default time to live of records in the zone, in seconds. ExternalDNS and cert-manager both write records here, so a short value keeps changes visible quickly." +} + +variable "contact_email" { + type = string + nullable = false + default = "" + description = "Contact address stored on the zone. Leave empty to let STACKIT pick its own default." +} + +variable "zone_description" { + type = string + nullable = false + default = "" + description = "Description stored on the zone. Leave empty to store none." +} + +# ── Records ──────────────────────────────────────────────────────────────────── + +variable "records" { + type = map(object({ + type = string + records = list(string) + ttl = optional(number) + comment = optional(string) + })) + nullable = false + default = {} + + description = <<-EOT + Record sets to create in the zone, keyed by the name relative to the zone. A key of `cluster1` in + the zone `likvid.stackit.run` gives `cluster1.likvid.stackit.run`, and `*.cluster1` gives the + wildcard below it. + + A value that is itself a domain name β€” the target of a `CNAME`, `MX` or `NS` record β€” must end in + a dot. STACKIT relativises a value without one against the zone, so `example.com` is stored as + `example.com.likvid.stackit.run.`. + + ```hcl + records = { + "cluster1" = { type = "A", records = ["203.0.113.17"] } + "*.cluster1" = { type = "A", records = ["203.0.113.17"] } + } + ``` + EOT + + validation { + condition = alltrue([for r in var.records : length(r.records) > 0]) + error_message = "Every record set must carry at least one value in `records`." + } + + validation { + condition = alltrue([for r in var.records : contains(["A", "AAAA", "ALIAS", "CAA", "CNAME", "DNAME", "MX", "NS", "PTR", "SRV", "TXT"], r.type)]) + error_message = "Each record set `type` must be one of A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SRV, TXT." + } +} + +# ── Delegation β€” customer-owned domains only ─────────────────────────────────── + +variable "delegation" { + type = object({ + parent_zone_project_id = string + parent_zone_name = string + nameservers = optional(list(string), ["ns1.stackit.cloud.", "ns2.stackit.zone."]) + ttl = optional(number, 3600) + }) + nullable = true + default = null + + description = <<-EOT + Write the `NS` record that delegates this zone from a parent zone in another STACKIT project. + Leave unset, which is the default and the usual case. + + **This works only for a domain the customer owns.** Under a free STACKIT suffix such as + `stackit.run` the zone itself cannot be created, so the delegation has nothing to point at β€” see + the API error quoted in main.tf. The module refuses that combination. + + `nameservers` must carry trailing dots. STACKIT relativises a value without one against the zone, + so `ns1.stackit.cloud` is stored as `ns1.stackit.cloud..` and the delegation points nowhere. + EOT + + validation { + condition = var.delegation == null ? true : !endswith(var.delegation.parent_zone_name, "stackit.run") + error_message = "Delegation under stackit.run is not possible. STACKIT allows exactly one label below a free subdomain and rejects the subzone with \"subdomain '' should only have one level\", so the NS record would delegate a name that cannot exist. Use a domain you own, or drop `delegation` and put the names into `records`." + } + + validation { + condition = var.delegation == null ? true : length(var.delegation.nameservers) > 0 + error_message = "delegation.nameservers must contain at least one nameserver, otherwise the delegated zone is not reachable. STACKIT requires both ns1.stackit.cloud. and ns2.stackit.zone. for its SLA to hold." + } +} + +# ── DNS credential ───────────────────────────────────────────────────────────── + +variable "dns_service_account_enabled" { + type = bool + nullable = false + default = true + description = "Create a service account with `dns.admin` on the zone's project and a key for it. cert-manager's DNS-01 solver and ExternalDNS both authenticate with that key. Turn it off when the consumer manages records through Terraform only, or when the backplane identity may not create service accounts." +} + +variable "dns_service_account_name" { + type = string + nullable = true + default = null + description = "Name of the DNS service account. Defaults to `mesh-dns-`, which keeps two zones in the same project apart." +} + +variable "dns_service_account_key_ttl_days" { + type = number + nullable = true + default = null + description = "Validity of the DNS service account key in days. Leave unset to create a key that stays valid until it is deleted. A key that expires has to be rotated by re-applying the building block before a certificate can renew." +} diff --git a/modules/stackit/dns/buildingblock/versions.tf b/modules/stackit/dns/buildingblock/versions.tf new file mode 100644 index 00000000..7f9b1189 --- /dev/null +++ b/modules/stackit/dns/buildingblock/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.11.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + # 0.110.0 is the version this module was written against. It carries the DNS zone data source + # with lookup by `dns_name` and the `iam` experiment that gates + # `stackit_authorization_project_role_assignment`. + version = ">= 0.110.0" + } + } +} diff --git a/modules/stackit/dns/meshstack_integration.tf b/modules/stackit/dns/meshstack_integration.tf new file mode 100644 index 00000000..af66de2d --- /dev/null +++ b/modules/stackit/dns/meshstack_integration.tf @@ -0,0 +1,316 @@ +variable "stackit_organization_id" { + type = string + description = "STACKIT organization ID under which every project the building block writes into lives." +} + +variable "stackit_project_id" { + type = string + description = "STACKIT project ID where the backplane service account will be created." +} + +variable "stackit_service_account_name" { + type = string + default = null + description = "Name of the backplane service account. Defaults to 'mesh-dns'. Override when deploying multiple backplane instances in the same STACKIT project." +} + +variable "stackit_additional_organization_roles" { + type = list(string) + default = [] + + description = <<-EOT + Extra STACKIT roles granted to the backplane service account at organization scope. The building + block creates a service account and a key for cert-manager and ExternalDNS, and STACKIT's + predefined role for creating service accounts could not be established, so the backplane does not + name one. Put the role your organization uses here, or leave the list empty and set + `stackit_dns_service_account_enabled` to false. + EOT +} + +variable "stackit_region" { + type = string + default = "eu01" + description = "STACKIT region used as the provider's default. STACKIT DNS itself is global." +} + +variable "stackit_dns_service_account_enabled" { + type = bool + default = true + description = "Create a service account with `dns.admin` on the zone's project and a key for it. cert-manager's DNS-01 solver and ExternalDNS both authenticate with that key." +} + +variable "stackit_dns_zone_default_ttl" { + type = number + default = 300 + description = "Default time to live offered for records in an ordered zone, in seconds." +} + +variable "meshstack" { + type = object({ + owning_workspace_identifier = string + tags = optional(map(list(string)), {}) + }) + description = "Shared meshStack context." +} + +variable "hub" { + type = object({ + git_ref = optional(string, "main") + bbd_draft = optional(bool, true) + }) + const = true + default = { + git_ref = "main" + bbd_draft = true + } + description = <<-EOT + `git_ref`: Hub release reference. Set to a tag (e.g. 'v1.2.3') or branch or commit sha of meshcloud/meshstack-hub repo. + `bbd_draft`: If true, allows changing the building block definition for upgrading dependent building blocks. + EOT +} + +output "building_block_definition" { + description = "BBD is consumed in building block compositions." + value = { + uuid = meshstack_building_block_definition.this.metadata.uuid + version_ref = var.hub.bbd_draft ? meshstack_building_block_definition.this.version_latest : meshstack_building_block_definition.this.version_latest_release + } +} + +data "meshstack_integrations" "integrations" {} + +module "backplane" { + source = "github.com/meshcloud/meshstack-hub//modules/stackit/dns/backplane?ref=${var.hub.git_ref}" + + project_id = var.stackit_project_id + organization_id = var.stackit_organization_id + service_account_name = coalesce(var.stackit_service_account_name, "mesh-dns") + additional_organization_roles = var.stackit_additional_organization_roles + + workload_identity_federation = { + issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer + subjects = [ + "${trimsuffix(data.meshstack_integrations.integrations.workload_identity_federation.replicator.subject, ":replicator")}:workspace.${var.meshstack.owning_workspace_identifier}.buildingblockdefinition.${meshstack_building_block_definition.this.metadata.uuid}" + ] + } +} + +resource "meshstack_building_block_definition" "this" { + metadata = { + owned_by_workspace = var.meshstack.owning_workspace_identifier + tags = var.meshstack.tags + } + + spec = { + display_name = "STACKIT DNS Zone" + symbol = "https://raw.githubusercontent.com/meshcloud/meshstack-hub/${var.hub.git_ref}/modules/stackit/dns/buildingblock/logo.png" + description = "Creates a STACKIT DNS zone with its record sets and a service account key that lets cert-manager and ExternalDNS manage records at runtime." + support_url = "https://portal.stackit.cloud" + target_type = "TENANT_LEVEL" + run_transparency = true + supported_platforms = [{ name = "STACKIT" }] + readme = chomp(<<-EOT + This building block creates a **public DNS zone** in your STACKIT project, the records you ask + for inside it, and a service account key your cluster uses to manage further records on its + own. + + ## 🎯 When to use it + + Use this building block when you: + - Need public DNS names for a workload and want to manage the records yourself. + - Run ExternalDNS or the SKE DNS extension in a cluster and want it to publish hostnames + automatically. + - Want a TLS certificate issued over the ACME DNS-01 challenge, including a wildcard. + + ## πŸ’‘ Usage examples + + **Example 1: DNS for an SKE cluster** + A team orders the zone `myteam.stackit.run` and enables the SKE DNS extension with the zone as + its filter. Every Ingress the team creates then gets its hostname published automatically. + + **Example 2: A wildcard certificate** + The building block returns a STACKIT service account key. cert-manager solves the ACME DNS-01 + challenge with it and issues one wildcard certificate for the zone, so a new application needs + no certificate request at all. + + ## πŸ“ A free STACKIT subdomain is one label deep + + `myteam.stackit.run` is a zone. `app.myteam.stackit.run` is **not** β€” STACKIT rejects a zone + that deep with *"subdomain should only have one level"*. Everything below your zone is a + record inside it, which is exactly what ExternalDNS and cert-manager create. Order a zone per + team, not a zone per application. + + ## πŸ”‘ The DNS key + + The key is the raw `sa.json` that the STACKIT cert-manager webhook expects. It carries the + `dns.admin` role on your STACKIT project, so it can write **every record in every zone of that + project** β€” it cannot be narrowed to one name. Hand it only to workloads you would trust with + the whole zone, keep it secret, and order the building block again to rotate it. + + ## πŸ“Š Shared Responsibility + + | Responsibility | Platform Team | Application Team | + |---|:---:|:---:| + | Provide the backplane identity used to create the zone | βœ… | ❌ | + | Run the STACKIT DNS service and its nameservers | βœ… | ❌ | + | Choose the zone name and the record TTLs | ❌ | βœ… | + | Create and remove the records inside the zone | ❌ | βœ… | + | Keep the returned service account key secret | ❌ | βœ… | + | Avoid overwriting records another workload owns | ❌ | βœ… | + EOT + ) + } + + version_spec = { + draft = var.hub.bbd_draft + deletion_mode = "DELETE" + + implementation = { + terraform = { + terraform_version = "1.11.0" + repository_url = "https://github.com/meshcloud/meshstack-hub.git" + repository_path = "modules/stackit/dns/buildingblock" + ref_name = var.hub.git_ref + async = false + use_mesh_http_backend_fallback = true + } + } + + inputs = { + project_id = { + display_name = "STACKIT Project ID" + description = "STACKIT project ID of the existing project the zone will be created in." + type = "STRING" + assignment_type = "PLATFORM_TENANT_ID" + } + + service_account_email = { + display_name = "Service Account Email" + description = "Email of the STACKIT service account for WIF-based authentication." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(module.backplane.service_account_email) + } + + stackit_region = { + display_name = "STACKIT Region" + description = "STACKIT region used as the provider's default." + type = "STRING" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_region) + } + + STACKIT_USE_OIDC = { + display_name = "STACKIT Use OIDC" + description = "Enables OIDC-based WIF for the STACKIT provider." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("1") + } + + STACKIT_FEDERATED_TOKEN_FILE = { + display_name = "STACKIT Federated Token File" + description = "Path to the WIF token file injected by meshStack." + type = "STRING" + assignment_type = "STATIC" + is_environment = true + argument = jsonencode("/var/run/secrets/workload-identity/azure/token") + } + + dns_service_account_enabled = { + display_name = "Create DNS Service Account" + description = "Create a service account with dns.admin on this project and a key for cert-manager and ExternalDNS." + type = "BOOLEAN" + assignment_type = "STATIC" + argument = jsonencode(var.stackit_dns_service_account_enabled) + } + + zone_name = { + display_name = "Zone Name" + description = "DNS name of the zone, for example 'myteam.stackit.run'. A free STACKIT subdomain carries exactly one label β€” put deeper names into records." + type = "STRING" + assignment_type = "USER_INPUT" + value_validation_regex = "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$" + validation_regex_error_message = "The zone name must be a lowercase domain name with at least two labels and no trailing dot, for example myteam.stackit.run." + } + + records = { + display_name = "Records" + description = "JSON object of record sets, keyed by the name relative to the zone. Example: {\"www\": {\"type\": \"A\", \"records\": [\"203.0.113.17\"]}}. Leave as {} to let ExternalDNS write the records instead." + type = "CODE" + assignment_type = "USER_INPUT" + default_value = jsonencode(jsonencode({})) + } + + zone_default_ttl = { + display_name = "Default Record TTL (seconds)" + description = "Default time to live of records inside the zone, in seconds." + type = "INTEGER" + assignment_type = "USER_INPUT" + default_value = jsonencode(var.stackit_dns_zone_default_ttl) + } + + contact_email = { + display_name = "Contact Email" + description = "Contact address stored on the zone. Leave empty to let STACKIT pick its own default." + type = "STRING" + assignment_type = "USER_INPUT" + default_value = jsonencode("") + } + } + + outputs = { + zone_name = { + display_name = "Zone Name" + type = "STRING" + assignment_type = "NONE" + } + + zone_id = { + display_name = "Zone ID" + type = "STRING" + assignment_type = "NONE" + } + + zone_project_id = { + display_name = "Zone Project ID" + type = "STRING" + assignment_type = "NONE" + } + + dns_service_account_email = { + display_name = "DNS Service Account" + type = "STRING" + assignment_type = "NONE" + } + + dns_service_account_key = { + display_name = "DNS Service Account Key" + type = "STRING" + assignment_type = "NONE" + } + + summary = { + display_name = "Summary" + type = "STRING" + assignment_type = "SUMMARY" + } + } + } +} + +terraform { + required_version = ">= 1.12.0" + + required_providers { + meshstack = { + source = "meshcloud/meshstack" + version = ">= 0.21.0" + } + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.110.0" + } + } +} From 9eb69d2788a08b32966622dfdc84c306a16bfa90 Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Thu, 13 Aug 2026 00:04:04 +0200 Subject: [PATCH 29/60] feat(stackit/model-serving): add the backplane that mints Model Serving tokens The module declared requiresBackplane: false because the STACKIT role name that permits minting Model Serving tokens could not be established. STACKIT's authorization API answers that question directly, so the backplane now exists. The role is model-serving.editor. It is the narrowest predefined role granting model-serving.token.create, which is the single permission the building block needs. Note the hyphen: the Terraform resource and the CLI namespace write the service as modelserving, the role does not, the same split PostgreSQL Flex has between postgresflex and postgres-flex.admin. The role assignment sits at folder scope rather than the organization scope the sibling backplanes use. STACKIT offers a different set of roles per resource type and no model-serving role appears on an organization, so an organization role assignment could not work here. A folder still covers every project below it, which keeps the property that matters for a TENANT_LEVEL building block: the platform team deploys the backplane before it knows the tenant projects. Co-Authored-By: Claude Opus 5 (1M context) --- .../stackit/model-serving/backplane/README.md | 116 ++++++++++++++++++ .../stackit/model-serving/backplane/main.tf | 43 +++++++ .../model-serving/backplane/outputs.tf | 4 + .../model-serving/backplane/variables.tf | 27 ++++ .../model-serving/backplane/versions.tf | 10 ++ .../model-serving/buildingblock/README.md | 8 +- .../model-serving/meshstack_integration.tf | 40 +++++- 7 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 modules/stackit/model-serving/backplane/README.md create mode 100644 modules/stackit/model-serving/backplane/main.tf create mode 100644 modules/stackit/model-serving/backplane/outputs.tf create mode 100644 modules/stackit/model-serving/backplane/variables.tf create mode 100644 modules/stackit/model-serving/backplane/versions.tf diff --git a/modules/stackit/model-serving/backplane/README.md b/modules/stackit/model-serving/backplane/README.md new file mode 100644 index 00000000..20b407e7 --- /dev/null +++ b/modules/stackit/model-serving/backplane/README.md @@ -0,0 +1,116 @@ +# STACKIT AI Model Serving – Backplane + +This module sets up the shared backplane configuration for the STACKIT AI Model Serving building +block. It creates a dedicated service account with a Workload Identity Federation (WIF) identity +provider and the permission required to issue Model Serving tokens in any project under a given +folder: + +- **`model-serving.editor`** β€” allows creating, reading and deleting Model Serving tokens and instances. + +Authentication uses WIF (OIDC token exchange) β€” no long-lived service account key is created or stored. + +## Why this role + +The building block creates one resource, `stackit_modelserving_token`, so the single permission it +needs is `model-serving.token.create`. Two predefined roles grant that permission, +`model-serving.editor` and `model-serving.admin`, and STACKIT's live role list gives both the same +ten permissions. This module picks `model-serving.editor`, because the two roles cost the same and +the editor description stays within managing resources, while the admin description also claims +"full administrative access to all model-serving resources and operations". + +Note the spelling. The Terraform resource and the CLI namespace write the service as one word, +`modelserving`, but the role name uses a hyphen: `model-serving.editor`. The same split exists for +PostgreSQL Flex, where the CLI namespace is `postgresflex` and the role is `postgres-flex.admin`. + +## Why the role is assigned at folder scope + +The building block is `TENANT_LEVEL`, so it takes its `project_id` from `PLATFORM_TENANT_ID` and +creates the token in the tenant's own STACKIT project. The platform team deploys this backplane +long before it knows which projects tenants will order into, so the role cannot be scoped to a +single project. + +The sibling backplanes solve this by granting their role at organization scope, but that is not +available here. STACKIT's authorization API offers a different set of roles per resource type, and +no model-serving role appears on an organization. These three calls establish it, each returning +`HTTP/1.1 200 OK`: + +```sh +stackit curl https://authorization.api.stackit.cloud/v2/organization//roles +# resourceType=organization, 76 roles, no model-serving role + +stackit curl https://authorization.api.stackit.cloud/v2/folder//roles +# resourceType=folder, 184 roles, including model-serving.editor, .admin and .reader + +stackit curl https://authorization.api.stackit.cloud/v2/project//roles +# resourceType=project, 182 roles, including model-serving.editor, .admin and .reader +``` + +The organization response is not filtered down to organization-management roles, so the absence is +a real one rather than an artifact of the call: `iaas.network.admin` is present at organization +scope, which is the role `modules/stackit/network/backplane` grants there. + +A folder covers every project below it, so folder scope keeps the property that makes organization +scope attractive: the platform team grants the role once, and every project a tenant later receives +is covered. Grant the role on the folder that holds the tenant projects. When tenant projects are +spread over several folders, deploy one backplane instance per folder and override +`service_account_name`. + +## Prerequisites + +- A STACKIT project where the service account will be created. +- A STACKIT service account with permissions to manage service accounts and folder-level role assignments. +- The STACKIT folder ID under which the tenant projects live. Projects placed directly under the organization are not covered. +- meshStack WIF issuer and subject from `data.meshstack_integrations.integrations`. +- The STACKIT provider configuration of the caller has to set `experiments = ["iam"]`, because the role assignment resources sit behind that provider experiment. + +## Usage + +```hcl +module "model_serving_backplane" { + source = "./backplane" + + project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + folder_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + + workload_identity_federation = { + issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer + subjects = [""] + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.11.0 | +| [stackit](#requirement\_stackit) | >= 0.110.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [stackit_authorization_folder_role_assignment.model_serving_editor](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_folder_role_assignment) | resource | +| [stackit_service_account.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account) | resource | +| [stackit_service_account_federated_identity_provider.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account_federated_identity_provider) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [folder\_id](#input\_folder\_id) | STACKIT folder ID under which the tenant projects live. The service account is granted 'model-serving.editor' on this folder, which covers every project below it. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | STACKIT project ID where the service account will be created. | `string` | n/a | yes | +| [service\_account\_name](#input\_service\_account\_name) | Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project. | `string` | `"mesh-model-serving"` | no | +| [workload\_identity\_federation](#input\_workload\_identity\_federation) | WIF issuer URL and subject list for the meshStack building block identity provider. | 

object({
    issuer   = string
    subjects = list(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [service\_account\_email](#output\_service\_account\_email) | Email of the STACKIT service account used by the buildingblock provider via WIF. | + diff --git a/modules/stackit/model-serving/backplane/main.tf b/modules/stackit/model-serving/backplane/main.tf new file mode 100644 index 00000000..786d9b08 --- /dev/null +++ b/modules/stackit/model-serving/backplane/main.tf @@ -0,0 +1,43 @@ +resource "stackit_service_account" "building_block" { + project_id = var.project_id + name = var.service_account_name +} + +resource "stackit_service_account_federated_identity_provider" "building_block" { + for_each = { for i, s in var.workload_identity_federation.subjects : tostring(i) => s } + + project_id = var.project_id + service_account_email = stackit_service_account.building_block.email + name = "meshstack-${each.key}" + issuer = var.workload_identity_federation.issuer + + assertions = [ + { + item = "aud" + operator = "equals" + value = "api://AzureADTokenExchange" + }, + { + item = "sub" + operator = "equals" + value = each.value + } + ] +} + +# model-serving.editor is the narrowest predefined role that grants `model-serving.token.create`, +# which is the one permission the building block needs. The only other predefined role that grants +# it is model-serving.admin, and STACKIT's live role list gives both roles the same permission set, +# so editor costs nothing and keeps the wider "administrative operations" wording out of the grant. +# +# The role is assigned at folder scope rather than organization scope. The building block is +# `TENANT_LEVEL`, so it reads its `project_id` from `PLATFORM_TENANT_ID` and the platform team +# deploys this backplane long before it knows which projects tenants will order into. A folder +# covers every project below it, so it keeps that property. Organization scope is not an option +# here: STACKIT's authorization API offers no model-serving role on an organization, only on +# folders and projects β€” see backplane/README.md for the calls that establish this. +resource "stackit_authorization_folder_role_assignment" "model_serving_editor" { + resource_id = var.folder_id + role = "model-serving.editor" + subject = stackit_service_account.building_block.email +} diff --git a/modules/stackit/model-serving/backplane/outputs.tf b/modules/stackit/model-serving/backplane/outputs.tf new file mode 100644 index 00000000..3fde57dd --- /dev/null +++ b/modules/stackit/model-serving/backplane/outputs.tf @@ -0,0 +1,4 @@ +output "service_account_email" { + value = stackit_service_account.building_block.email + description = "Email of the STACKIT service account used by the buildingblock provider via WIF." +} diff --git a/modules/stackit/model-serving/backplane/variables.tf b/modules/stackit/model-serving/backplane/variables.tf new file mode 100644 index 00000000..d8aadcd3 --- /dev/null +++ b/modules/stackit/model-serving/backplane/variables.tf @@ -0,0 +1,27 @@ +variable "project_id" { + type = string + nullable = false + description = "STACKIT project ID where the service account will be created." +} + +variable "folder_id" { + type = string + nullable = false + description = "STACKIT folder ID under which the tenant projects live. The service account is granted 'model-serving.editor' on this folder, which covers every project below it." +} + +variable "workload_identity_federation" { + type = object({ + issuer = string + subjects = list(string) + }) + nullable = false + description = "WIF issuer URL and subject list for the meshStack building block identity provider." +} + +variable "service_account_name" { + type = string + default = "mesh-model-serving" + nullable = false + description = "Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project." +} diff --git a/modules/stackit/model-serving/backplane/versions.tf b/modules/stackit/model-serving/backplane/versions.tf new file mode 100644 index 00000000..5c2057d3 --- /dev/null +++ b/modules/stackit/model-serving/backplane/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.11.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.110.0" + } + } +} diff --git a/modules/stackit/model-serving/buildingblock/README.md b/modules/stackit/model-serving/buildingblock/README.md index 828eb257..da0e4341 100644 --- a/modules/stackit/model-serving/buildingblock/README.md +++ b/modules/stackit/model-serving/buildingblock/README.md @@ -3,9 +3,6 @@ name: STACKIT AI Model Serving Access supportedPlatforms: - stackit description: Issues a scoped STACKIT AI Model Serving API token so a tenant can call the sovereign LLM API. -# The module creates the token in a STACKIT project that already exists and receives the service -# account it authenticates with as an input, so there is no cloud-side setup to perform ahead of time. -requiresBackplane: false --- # STACKIT AI Model Serving Access Building Block @@ -23,6 +20,11 @@ endpoint as a LiteLLM backend needs nothing beyond the `api_base` and `token` ou STACKIT parses `ttl_duration` with Go's duration parser, which knows no day unit. Write 90 days as `2160h`; a value such as `90d` is rejected. +The module receives the service account it authenticates with as an input and reaches STACKIT +through workload identity federation, so it stores no long-lived key. The `backplane/` tier creates +that service account and grants it `model-serving.editor` on the folder holding the tenant projects +β€” see [backplane/README.md](../backplane/README.md) for the role and the scope. + ## Requirements diff --git a/modules/stackit/model-serving/meshstack_integration.tf b/modules/stackit/model-serving/meshstack_integration.tf index 93f933c0..d1fb248c 100644 --- a/modules/stackit/model-serving/meshstack_integration.tf +++ b/modules/stackit/model-serving/meshstack_integration.tf @@ -1,6 +1,17 @@ -variable "stackit_service_account_email" { +variable "stackit_project_id" { type = string - description = "Email of the STACKIT service account the building block authenticates with via workload identity federation. The account needs permission to create AI Model Serving tokens in the tenant projects this definition targets." + description = "STACKIT project ID where the backplane service account will be created." +} + +variable "stackit_folder_id" { + type = string + description = "STACKIT folder ID under which the tenant projects live. The backplane grants the service account 'model-serving.editor' on this folder, which covers every project below it." +} + +variable "stackit_service_account_name" { + type = string + default = null + description = "Name of the backplane service account. Defaults to 'mesh-model-serving'. Override when deploying multiple backplane instances in the same STACKIT project." } variable "stackit_region" { @@ -49,6 +60,23 @@ output "building_block_definition" { } } +data "meshstack_integrations" "integrations" {} + +module "backplane" { + source = "github.com/meshcloud/meshstack-hub//modules/stackit/model-serving/backplane?ref=${var.hub.git_ref}" + + project_id = var.stackit_project_id + folder_id = var.stackit_folder_id + service_account_name = coalesce(var.stackit_service_account_name, "mesh-model-serving") + + workload_identity_federation = { + issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer + subjects = [ + "${trimsuffix(data.meshstack_integrations.integrations.workload_identity_federation.replicator.subject, ":replicator")}:workspace.${var.meshstack.owning_workspace_identifier}.buildingblockdefinition.${meshstack_building_block_definition.this.metadata.uuid}" + ] + } +} + resource "meshstack_building_block_definition" "this" { metadata = { owned_by_workspace = var.meshstack.owning_workspace_identifier @@ -105,7 +133,7 @@ resource "meshstack_building_block_definition" "this" { | Responsibility | Platform Team | Application Team | |---|:---:|:---:| - | Enable STACKIT AI Model Serving and provide the service account that issues tokens | βœ… | ❌ | + | Enable STACKIT AI Model Serving and provide the backplane identity that issues tokens | βœ… | ❌ | | Set the token lifetime that applies to every project | βœ… | ❌ | | Store the token in the application's own secret store | ❌ | βœ… | | Choose the model and carry the cost of the calls | ❌ | βœ… | @@ -142,7 +170,7 @@ resource "meshstack_building_block_definition" "this" { description = "Email of the STACKIT service account for WIF-based authentication." type = "STRING" assignment_type = "STATIC" - argument = jsonencode(var.stackit_service_account_email) + argument = jsonencode(module.backplane.service_account_email) } STACKIT_USE_OIDC = { @@ -247,5 +275,9 @@ terraform { source = "meshcloud/meshstack" version = ">= 0.21.0" } + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.110.0" + } } } From 80f9bee983661075ef53db24e0edeabfd71b4c4b Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Thu, 13 Aug 2026 00:04:19 +0200 Subject: [PATCH 30/60] fix(kubernetes/ingress): pin the STACKIT webhook chart to a version the index serves helm_release resolves a chart version through the published index, and the index at https://stackitcloud.github.io/stackit-cert-manager-webhook ends at 0.4.9. The pin of 0.4.10 came from the GitHub release tag stackit-cert-manager-webhook-0.4.10, which exists but was never added to the index, so the dns01.stackit path would have failed at apply with "no chart version found". Verified with: helm show chart stackit-cert-manager-webhook \ --repo https://stackitcloud.github.io/stackit-cert-manager-webhook \ --version 0.4.10 Error: chart "stackit-cert-manager-webhook" version "0.4.10" not found The same command resolves 0.4.9. The comment on the variable records that the default follows the index rather than the release tags. Co-Authored-By: Claude Opus 5 (1M context) --- modules/kubernetes/ingress/buildingblock/README.md | 2 +- modules/kubernetes/ingress/buildingblock/variables.tf | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/kubernetes/ingress/buildingblock/README.md b/modules/kubernetes/ingress/buildingblock/README.md index c30b1e75..48ae86cd 100644 --- a/modules/kubernetes/ingress/buildingblock/README.md +++ b/modules/kubernetes/ingress/buildingblock/README.md @@ -209,7 +209,7 @@ No modules. | [haproxy\_version](#input\_haproxy\_version) | Version of the haproxytech/kubernetes-ingress Helm chart. See https://github.com/haproxytech/helm-charts/blob/main/kubernetes-ingress/Chart.yaml. | `string` | `"1.49.0"` | no | | [ingress\_class\_name](#input\_ingress\_class\_name) | Name of the IngressClass the controller serves. The HTTP-01 solver of the ClusterIssuer uses the same name. | `string` | `"haproxy"` | no | | [stackit\_webhook\_resources](#input\_stackit\_webhook\_resources) | Resource requests and limits of the STACKIT cert-manager webhook. Only used when dns01.stackit
is set. The default is sized for a demonstration cluster and a production consumer has to raise
it.

The chart sets no resources and its values file states that `100m` CPU and `128Mi` memory are
enough for the webhook, which is what the limit uses. The webhook answers one DNS-01 challenge
per certificate renewal, so the request stays well below that. Production wants the chart's own
figures as the request as well. | 
object({
    requests = optional(object({ cpu = optional(string), memory = optional(string) }), {})
    limits   = optional(object({ cpu = optional(string), memory = optional(string) }), {})
  })
| 
{
  "limits": {
    "cpu": "100m",
    "memory": "128Mi"
  },
  "requests": {
    "cpu": "10m",
    "memory": "64Mi"
  }
}
| no | -| [stackit\_webhook\_version](#input\_stackit\_webhook\_version) | Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook. | `string` | `"0.4.10"` | no | +| [stackit\_webhook\_version](#input\_stackit\_webhook\_version) | Version of the stackit-cert-manager-webhook Helm chart. Must be a version served by the chart index at https://stackitcloud.github.io/stackit-cert-manager-webhook, which lags behind the GitHub release tags. Only used when dns01.stackit is set. | `string` | `"0.4.9"` | no | | [wildcard\_certificate\_name](#input\_wildcard\_certificate\_name) | Name of the wildcard Certificate and of the secret it writes, both in haproxy\_namespace. Only used when dns01 is set. | `string` | `"wildcard-tls"` | no | ## Outputs diff --git a/modules/kubernetes/ingress/buildingblock/variables.tf b/modules/kubernetes/ingress/buildingblock/variables.tf index 138805e5..5c603e81 100644 --- a/modules/kubernetes/ingress/buildingblock/variables.tf +++ b/modules/kubernetes/ingress/buildingblock/variables.tf @@ -237,10 +237,16 @@ variable "wildcard_certificate_name" { description = "Name of the wildcard Certificate and of the secret it writes, both in haproxy_namespace. Only used when dns01 is set." } +# The default follows the published chart index rather than the GitHub release tags, because the +# two diverged: the repository tagged a `stackit-cert-manager-webhook-0.4.10` release, but +# https://stackitcloud.github.io/stackit-cert-manager-webhook/index.yaml was never regenerated and +# still ends at 0.4.9. `helm_release` resolves the version through that index, so pinning 0.4.10 +# fails at apply with "no chart version found". Raise this default only after the index serves the +# newer version. variable "stackit_webhook_version" { type = string - default = "0.4.10" - description = "Version of the stackit-cert-manager-webhook Helm chart. Only used when dns01.stackit is set. See https://github.com/stackitcloud/stackit-cert-manager-webhook." + default = "0.4.9" + description = "Version of the stackit-cert-manager-webhook Helm chart. Must be a version served by the chart index at https://stackitcloud.github.io/stackit-cert-manager-webhook, which lags behind the GitHub release tags. Only used when dns01.stackit is set." } variable "stackit_webhook_resources" { From 096812a5032bc974d350ddbf2ca5c06c2b7c28ff Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Thu, 13 Aug 2026 00:11:55 +0200 Subject: [PATCH 31/60] fix(stackit/postgresflex): grant postgres-flex.admin at folder scope The backplane assigned postgres-flex.admin at organization scope, which would have failed at apply. STACKIT offers a different set of roles per resource type and no postgres-flex role exists on an organization: the authorization API returns 76 roles for resourceType=organization and none of them belongs to postgres-flex, while the folder and project lists return 184 and 182 roles and both contain postgres-flex.admin. iaas.network.admin is present at organization scope, so the organization list is complete for its resource type and the absence is real. modules/stackit/network/backplane grants iaas.network.admin at organization scope and is correct for that reason; the pattern does not carry over to a product role. The role now sits at folder scope, the same move modules/stackit/model-serving/backplane already made for model-serving.editor. A folder covers every project below it, so it keeps the property the building block needs: it is TENANT_LEVEL and reads its project_id from PLATFORM_TENANT_ID, so the platform team grants the role before it knows which projects tenants will order into. The backplane takes folder_id instead of organization_id and meshstack_integration.tf passes stackit_folder_id. The README now records why the role is postgres-flex.admin rather than postgres-flex.editor: the building block definition sets deletion_mode DELETE, and editor grants none of the delete permissions for instances, databases and users. Co-Authored-By: Claude Opus 5 (1M context) --- .../stackit/postgresflex/backplane/README.md | 69 +++++++++++++++---- .../stackit/postgresflex/backplane/main.tf | 22 +++--- .../postgresflex/backplane/variables.tf | 4 +- .../postgresflex/meshstack_integration.tf | 10 +-- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/modules/stackit/postgresflex/backplane/README.md b/modules/stackit/postgresflex/backplane/README.md index 9f4988dd..5ebb6aca 100644 --- a/modules/stackit/postgresflex/backplane/README.md +++ b/modules/stackit/postgresflex/backplane/README.md @@ -3,29 +3,68 @@ This module sets up the shared backplane configuration for the STACKIT PostgreSQL Flex building block. It creates a dedicated service account with a Workload Identity Federation (WIF) identity provider and the permission required to create PostgreSQL Flex instances in any project under a -given organization: +given folder: - **`postgres-flex.admin`** β€” allows creating and managing PostgreSQL Flex instances, databases and users. Authentication uses WIF (OIDC token exchange) β€” no long-lived service account key is created or stored. -## Why the role is assigned at organization scope +## Why this role -The building block is `TENANT_LEVEL`, so it creates the instance in the tenant's own STACKIT project. -The platform team deploys this backplane once, long before it knows which projects future instances -will land in, so the role cannot be scoped to a single project. STACKIT offers no predefined role -narrower than `postgres-flex.admin` for creating instances. A custom role is the least-privilege -alternative once the exact permission set is stable. +The building block creates an instance, a database and a user, and its building block definition +sets `deletion_mode = "DELETE"`, so the service account also has to remove all three again. +`postgres-flex.editor` can create them but grants none of the delete permissions, so +`postgres-flex.admin` is the narrowest predefined role that covers the full lifecycle. A custom role +is the least-privilege alternative once the exact permission set is stable. -This mirrors `modules/stackit/network/backplane`, which grants `iaas.network.admin` at organization -scope for the same reason. +Note the spelling. The Terraform resources and the CLI namespace write the service as one word, +`postgresflex`, but the role name uses a hyphen: `postgres-flex.admin`. + +## Why the role is assigned at folder scope + +The building block is `TENANT_LEVEL`, so it takes its `project_id` from `PLATFORM_TENANT_ID` and +creates the instance in the tenant's own STACKIT project. The platform team deploys this backplane +long before it knows which projects tenants will order into, so the role cannot be scoped to a +single project. + +`modules/stackit/network/backplane` grants its role at organization scope, but that is not available +here. STACKIT's authorization API offers a different set of roles per resource type, and no +postgres-flex role appears on an organization. These three calls establish it, each returning +`HTTP/1.1 200 OK`: + +```sh +stackit curl https://authorization.api.stackit.cloud/v2/organization//roles +# resourceType=organization, 76 roles, no postgres-flex role + +stackit curl https://authorization.api.stackit.cloud/v2/folder//roles +# resourceType=folder, 184 roles, including postgres-flex.admin, .editor, .reader, +# .user-admin and .metrics-reader + +stackit curl https://authorization.api.stackit.cloud/v2/project//roles +# resourceType=project, 182 roles, including the same five postgres-flex roles +``` + +The organization response is not filtered down to organization-management roles, so the absence is +a real one rather than an artifact of the call: `iaas.network.admin` is present at organization +scope, which is the role `modules/stackit/network/backplane` grants there. An organization role +assignment for `postgres-flex.admin` therefore fails at apply. Do not move this grant back. + +A folder covers every project below it, so folder scope keeps the property that makes organization +scope attractive: the platform team grants the role once, and every project a tenant later receives +is covered. Grant the role on the folder that holds the tenant projects. When tenant projects are +spread over several folders, deploy one backplane instance per folder and override +`service_account_name`. + +The same reasoning applies to `modules/stackit/model-serving/backplane`, which grants +`model-serving.editor` at folder scope. ## Prerequisites - A STACKIT project where the service account will be created. -- A STACKIT service account with permissions to manage service accounts and organization-level role assignments. -- The STACKIT organization ID under which target projects live. +- A STACKIT service account with permissions to manage service accounts and folder-level role assignments. +- The STACKIT folder ID under which the tenant projects live. Projects placed directly under the organization are not covered. - meshStack WIF issuer and subject from `data.meshstack_integrations.integrations`. +- The STACKIT provider configuration of the caller has to set `experiments = ["iam"]`, because the role assignment resources sit behind that provider experiment. ## Usage @@ -33,8 +72,8 @@ scope for the same reason. module "postgresflex_backplane" { source = "./backplane" - project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" - organization_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + folder_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" workload_identity_federation = { issuer = data.meshstack_integrations.integrations.workload_identity_federation.replicator.issuer @@ -59,7 +98,7 @@ No modules. | Name | Type | |------|------| -| [stackit_authorization_organization_role_assignment.postgres_flex_admin](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_organization_role_assignment) | resource | +| [stackit_authorization_folder_role_assignment.postgres_flex_admin](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/authorization_folder_role_assignment) | resource | | [stackit_service_account.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account) | resource | | [stackit_service_account_federated_identity_provider.building_block](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/service_account_federated_identity_provider) | resource | @@ -67,7 +106,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [organization\_id](#input\_organization\_id) | STACKIT organization ID where the service account will be granted permissions to manage PostgreSQL Flex instances. | `string` | n/a | yes | +| [folder\_id](#input\_folder\_id) | STACKIT folder ID under which the tenant projects live. The service account is granted 'postgres-flex.admin' on this folder, which covers every project below it. | `string` | n/a | yes | | [project\_id](#input\_project\_id) | STACKIT project ID where the service account will be created. | `string` | n/a | yes | | [service\_account\_name](#input\_service\_account\_name) | Name of the service account created in the STACKIT project. Override when deploying multiple backplane instances in the same project. | `string` | `"mesh-postgresflex"` | no | | [workload\_identity\_federation](#input\_workload\_identity\_federation) | WIF issuer URL and subject list for the meshStack building block identity provider. | 
object({
    issuer   = string
    subjects = list(string)
  })
| n/a | yes | diff --git a/modules/stackit/postgresflex/backplane/main.tf b/modules/stackit/postgresflex/backplane/main.tf index 82a15079..6baad2be 100644 --- a/modules/stackit/postgresflex/backplane/main.tf +++ b/modules/stackit/postgresflex/backplane/main.tf @@ -25,14 +25,20 @@ resource "stackit_service_account_federated_identity_provider" "building_block" ] } -# postgres-flex.admin at org scope allows creating instances, users and databases in any tenant -# project under the organization. Required because this is a TENANT_LEVEL building block: the -# backplane is deployed once, before the target project of any future instance is known, so -# permissions can't be scoped to a single project ahead of time. STACKIT offers no narrower -# predefined role than postgres-flex.admin for creating instances; a custom role would be the -# least-privilege alternative once the exact permission set is stable. -resource "stackit_authorization_organization_role_assignment" "postgres_flex_admin" { - resource_id = var.organization_id +# postgres-flex.admin allows creating and deleting instances, databases and users. The building +# block runs with deletion_mode DELETE, so it needs the delete permissions as well, and +# postgres-flex.editor grants none of them: it can create an instance, a database and a user, but +# it cannot remove any of the three. A custom role would be the least-privilege alternative once +# the exact permission set is stable. +# +# The role is assigned at folder scope rather than organization scope. The building block is +# `TENANT_LEVEL`, so it reads its `project_id` from `PLATFORM_TENANT_ID` and the platform team +# deploys this backplane long before it knows which projects tenants will order into. A folder +# covers every project below it, so it keeps that property. Organization scope is not an option +# here: STACKIT's authorization API offers no postgres-flex role on an organization, only on +# folders and projects β€” see backplane/README.md for the calls that establish this. +resource "stackit_authorization_folder_role_assignment" "postgres_flex_admin" { + resource_id = var.folder_id role = "postgres-flex.admin" subject = stackit_service_account.building_block.email } diff --git a/modules/stackit/postgresflex/backplane/variables.tf b/modules/stackit/postgresflex/backplane/variables.tf index ff3d097a..2dd08d64 100644 --- a/modules/stackit/postgresflex/backplane/variables.tf +++ b/modules/stackit/postgresflex/backplane/variables.tf @@ -4,10 +4,10 @@ variable "project_id" { description = "STACKIT project ID where the service account will be created." } -variable "organization_id" { +variable "folder_id" { type = string nullable = false - description = "STACKIT organization ID where the service account will be granted permissions to manage PostgreSQL Flex instances." + description = "STACKIT folder ID under which the tenant projects live. The service account is granted 'postgres-flex.admin' on this folder, which covers every project below it." } variable "workload_identity_federation" { diff --git a/modules/stackit/postgresflex/meshstack_integration.tf b/modules/stackit/postgresflex/meshstack_integration.tf index cee078bd..fc161758 100644 --- a/modules/stackit/postgresflex/meshstack_integration.tf +++ b/modules/stackit/postgresflex/meshstack_integration.tf @@ -1,11 +1,11 @@ -variable "stackit_organization_id" { +variable "stackit_project_id" { type = string - description = "STACKIT organization ID under which target projects live." + description = "STACKIT project ID where the backplane service account will be created." } -variable "stackit_project_id" { +variable "stackit_folder_id" { type = string - description = "STACKIT project ID where the backplane service account will be created." + description = "STACKIT folder ID under which the tenant projects live. The backplane grants the service account 'postgres-flex.admin' on this folder, which covers every project below it." } variable "stackit_service_account_name" { @@ -70,7 +70,7 @@ module "backplane" { source = "github.com/meshcloud/meshstack-hub//modules/stackit/postgresflex/backplane?ref=${var.hub.git_ref}" project_id = var.stackit_project_id - organization_id = var.stackit_organization_id + folder_id = var.stackit_folder_id service_account_name = coalesce(var.stackit_service_account_name, "mesh-postgresflex") workload_identity_federation = { From 08c6ef6e8ce5ca31a83d86568b292b79fb1c3f2f Mon Sep 17 00:00:00 2001 From: Andreas Grub Date: Thu, 13 Aug 2026 00:18:39 +0200 Subject: [PATCH 32/60] feat(stackit/dns): add record sets to a zone the module does not own A free STACKIT subdomain admits exactly one label, so a subzone per cluster is impossible and every cluster has to share one platform-owned zone. The module gains `create_zone = false`, which skips the zone and writes its record sets into a zone that already exists, identified either by `zone_id` or by a lookup on `zone_name` through the `stackit_dns_zone` data source. The new `wildcard` input writes the record that sends every application hostname to the ingress controller. Its `label` is optional: unset puts the wildcard at the zone apex, which reproduces the record the three SKE foundations have today down to the comment, and a label gives `*.