feat: per-tenant Valkey ACL provisioning

ARCHITECTURE.md

@@ -494,6 +494,101 @@ graph TB
     Helm --> K8s
 ```
 
+---
+
+## Valkey ACL Provisioning
+
+Per-tenant cache isolation enforced at the Valkey protocol level.
+Each site namespace gets a dedicated ACL user restricted to `~{site}:*` key patterns.
+
+### Flow: Initial Provisioning
+
+```mermaid
+sequenceDiagram
+    participant GitOps as GitOps / Bootstrap
+    participant K8s as Kubernetes API
+    participant Ctrl as NamespaceReconciler
+    participant Valkey as Valkey (all nodes)
+    participant Webhook as Mutating Webhook
+    participant Pod as Site Pod
+
+    GitOps->>K8s: kubectl annotate ns sites-loja<br/>deco.sites/valkey-acl=true
+    K8s->>Ctrl: Reconcile Event
+    Ctrl->>Ctrl: Generate random password
+    Ctrl->>Valkey: ACL SETUSER loja on >pass ~loja:* ~lock:loja:*<br/>(master + all replicas via Sentinel discovery)
+    Ctrl->>K8s: Create Secret "valkey-acl" in sites-loja<br/>LOADER_CACHE_REDIS_USERNAME=loja<br/>LOADER_CACHE_REDIS_PASSWORD=<pass>
+    Ctrl->>K8s: Patch ksvc spec.template.annotations<br/>(triggers new Knative Revision)
+    K8s->>Webhook: Intercept ksvc update
+    Webhook->>K8s: Inject envFrom: secretRef: valkey-acl (optional=true)
+    K8s->>Pod: New pod starts with credentials
+    Pod->>Valkey: AUTH loja <pass> → reads/writes only ~loja:*
+```
+
+### Flow: Sentinel Failover Recovery
+
+```mermaid
+sequenceDiagram
+    participant Sentinel as Valkey Sentinel
+    participant Ctrl as NamespaceReconciler
+    participant PubSub as +switch-master channel
+    participant Valkey as New Master + Replicas
+
+    Note over Sentinel: Master fails
+    Sentinel->>Sentinel: Elect new master (quorum)
+    Sentinel->>PubSub: Publish +switch-master event
+    PubSub->>Ctrl: WatchFailover() receives event
+    Ctrl->>Ctrl: RecordSentinelFailover() metric++
+    Ctrl->>Ctrl: TriggerResyncAll() — patch all annotated namespaces
+    loop For each managed namespace
+        Ctrl->>Valkey: ACL SETUSER (master + all replicas)
+    end
+    Note over Ctrl: Recovery in seconds, not minutes
+```
+
+### ACL Replication Caveat
+
+Valkey does **not** replicate `ACL SETUSER` commands to replicas.
+The operator handles this by running ACL commands on every node individually:
+
+```
+UpsertUser(ctx, username, password)
+  ├── ACL SETUSER on master       (via Sentinel FailoverClient)
+  ├── ACL SETUSER on replica-1    (direct connection, discovered via SENTINEL REPLICAS)
+  └── ACL SETUSER on replica-N
+```
+
+This ensures all nodes (master and read replicas) are always in sync,
+which is required because site pods use a separate read-replica endpoint.
+
+### Periodic Resync
+
+The reconciler requeues every namespace on a configurable interval (default: 10min)
+to ensure ACLs stay in sync after any undetected node restart.
+
+```
+VALKEY_ACL_RESYNC_PERIOD=10m  # env var or --valkey-acl-resync-period flag
+```
+
+To force immediate resync of all managed namespaces:
+
+```bash
+kubectl annotate ns -l deco.sites/valkey-acl=true \
+  deco.sites/valkey-acl-sync=$(date +%s) --overwrite
+```
+
+### Metrics
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `deco_operator_valkey_acl_provisioned_total` | Counter | ACL users provisioned |
+| `deco_operator_valkey_acl_deleted_total` | Counter | ACL users deleted on namespace removal |
+| `deco_operator_valkey_acl_errors_total{operation}` | Counter | Errors by operation (upsert/delete/check) |
+| `deco_operator_valkey_acl_self_healed_total` | Counter | Re-provisions after ACL loss |
+| `deco_operator_valkey_tenants_provisioned` | Gauge | Current provisioned tenants (seeded on startup) |
+| `deco_operator_valkey_sentinel_failovers_total` | Counter | Sentinel +switch-master events detected |
+
+---
+
 ## Summary
 
 The Deco CMS Operator provides:
@@ -504,7 +599,6 @@ The Deco CMS Operator provides:
 4. **Reliable Notifications**: Parallel, time-bounded, with retries
 5. **Deterministic Updates**: Timestamp-based verification
 6. **Trackable Rollouts**: Watch conditions with commit/timestamp
-7. **Production Ready**: Handles scale, failures, and edge cases
-
-**All with zero configuration required from users!** 🚀
+7. **Per-tenant Cache Isolation**: Valkey ACL per site, enforced at protocol level
+8. **Production Ready**: Handles scale, failures, and edge cases
 

chart/templates/clusterrole-operator-manager-role.yaml

@@ -15,12 +15,30 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
   - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
   - secrets
   verbs:
+  - create
   - get
   - list
   - watch
@@ -57,4 +75,6 @@ rules:
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch

chart/templates/deployment-operator-controller-manager.yaml

@@ -31,11 +31,29 @@ spec:
         command:
         - /manager
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        {{- if .Values.github.token }}
+        {{- if or .Values.github.token .Values.valkey.sentinelUrls }}
         env:
+        {{- if .Values.github.token }}
         - name: GITHUB_TOKEN
           value: {{ .Values.github.token | quote }}
         {{- end }}
+        {{- if .Values.valkey.sentinelUrls }}
+        - name: VALKEY_SENTINEL_URLS
+          value: {{ .Values.valkey.sentinelUrls | quote }}
+        - name: VALKEY_SENTINEL_MASTER_NAME
+          value: {{ .Values.valkey.sentinelMasterName | quote }}
+        {{- if .Values.valkey.existingSecret }}
+        - name: VALKEY_ADMIN_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.valkey.existingSecret | quote }}
+              key: {{ .Values.valkey.existingSecretKey | quote }}
+        {{- else if .Values.valkey.adminPassword }}
+        - name: VALKEY_ADMIN_PASSWORD
+          value: {{ .Values.valkey.adminPassword | quote }}
+        {{- end }}
+        {{- end }}
+        {{- end }}
         livenessProbe:
           httpGet:
             path: /healthz

chart/values.yaml

@@ -15,6 +15,19 @@ replicaCount: 1
 github:
   token: ""
 
+# Valkey (Redis) ACL provisioning
+# When sentinelUrls is set, the operator provisions per-tenant ACL users in Valkey
+# for each namespace annotated with deco.sites/valkey-acl: "true".
+# Leave sentinelUrls empty to disable ACL provisioning (NoopClient).
+valkey:
+  sentinelUrls: ""
+  sentinelMasterName: "mymaster"
+  # adminPassword: inline value (not recommended for production).
+  # Use existingSecret + existingSecretKey to source from a K8s Secret instead.
+  adminPassword: ""
+  existingSecret: ""
+  existingSecretKey: "password"
+
 # Resource limits and requests
 resources:
   limits:

cmd/bootstrap/main.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// bootstrap is a one-shot CLI that annotates existing site Namespaces so the
+// NamespaceReconciler picks them up and provisions Valkey ACL credentials.
+//
+// Usage:
+//
+//	go run ./cmd/bootstrap --namespace-pattern sites-
+//	go run ./cmd/bootstrap --namespace-pattern sites- --dry-run
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+const valkeyACLAnnotation = "deco.sites/valkey-acl"
+
+func main() {
+	var namespacePattern string
+	var dryRun bool
+
+	flag.StringVar(&namespacePattern, "namespace-pattern", "sites-",
+		"Prefix used to filter Namespaces eligible for Valkey ACL provisioning.")
+	flag.BoolVar(&dryRun, "dry-run", false,
+		"Print which Namespaces would be annotated without making changes.")
+
+	opts := zap.Options{Development: true}
+	opts.BindFlags(flag.CommandLine)
+	flag.Parse()
+
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	log := ctrl.Log.WithName("bootstrap")
+
+	scheme := runtime.NewScheme()
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+
+	cfg, err := ctrl.GetConfig()
+	if err != nil {
+		log.Error(err, "Failed to get kubeconfig")
+		os.Exit(1)
+	}
+
+	k8s, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		log.Error(err, "Failed to create Kubernetes client")
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+
+	nsList := &corev1.NamespaceList{}
+	if err := k8s.List(ctx, nsList); err != nil {
+		log.Error(err, "Failed to list Namespaces")
+		os.Exit(1)
+	}
+
+	annotated, skipped := 0, 0
+	for i := range nsList.Items {
+		ns := &nsList.Items[i]
+
+		if !strings.HasPrefix(ns.Name, namespacePattern) {
+			continue
+		}
+
+		if ns.Annotations[valkeyACLAnnotation] == "true" {
+			log.Info("Already annotated, skipping", "namespace", ns.Name)
+			skipped++
+			continue
+		}
+
+		if dryRun {
+			fmt.Printf("[dry-run] would annotate namespace %s\n", ns.Name)
+			annotated++
+			continue
+		}
+
+		patch := client.MergeFrom(ns.DeepCopy())
+		if ns.Annotations == nil {
+			ns.Annotations = make(map[string]string)
+		}
+		ns.Annotations[valkeyACLAnnotation] = "true"
+		if err := k8s.Patch(ctx, ns, patch); err != nil {
+			log.Error(err, "Failed to annotate namespace", "namespace", ns.Name)
+			continue
+		}
+		log.Info("Annotated namespace", "namespace", ns.Name)
+		annotated++
+	}
+
+	log.Info("Bootstrap complete", "annotated", annotated, "skipped", skipped)
+}