feat: add STS web identity and stabilize live e2e

Cargo.toml

@@ -12,6 +12,7 @@ chrono = { version = "0.4", features = ["serde"] }
 const-str = "1.0.0"
 serde = { version = "1.0.228", features = ["derive"] }
 tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "io-std", "io-util"] }
+tokio-rustls = "0.26"
 tokio-stream = { version = "0.1", features = ["sync"] }
 tokio-util = { version = "0.7", features = ["io", "compat"] }
 futures = "0.3.31"
@@ -27,18 +28,26 @@ clap = { version = "4.5.54", features = ["derive"] }
 rustls = { version = "0.23", default-features = false, features = ["ring"] }
 rustls-pemfile = "2.2.0"
 webpki = { package = "rustls-webpki", version = "0.103" }
+rcgen = "0.13"
 sha2 = "0.10"
+hmac = "0.12"
+hex = "0.4"
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+url = "2.5"
 shadow-rs = "1.5.0"
 snafu = { version = "0.8.9", features = ["futures"] }
 
 # Console dependencies
 axum = { version = "0.7", features = ["macros", "json"] }
+hyper = "1"
+hyper-util = { version = "0.1", features = ["server-auto", "service", "tokio"] }
 tower = "0.5"
 tower-http = { version = "0.6", features = ["cors", "trace", "compression-gzip"] }
 jsonwebtoken = "9.3"
 http = "1.2"
 utoipa = { version = "5", features = ["chrono"] }
 utoipa-swagger-ui = { version = "8", features = ["axum", "vendored"] }
+async-trait = { version = "0.1.89", default-features = false }
 
 [dev-dependencies]
 
@@ -50,4 +59,4 @@ unused_variables = "allow"
 
 [lints.clippy]
 unwrap_used = "deny"
-expect_used = "deny"
+expect_used = "deny"

Makefile

@@ -15,7 +15,7 @@
 .PHONY: pre-commit fmt fmt-check clippy test build help
 .PHONY: docker-build-operator docker-build-console-web docker-build-all
 .PHONY: console-lint console-fmt console-fmt-check
-.PHONY: e2e-check e2e-live-create e2e-live-run e2e-live-update e2e-live-delete
+.PHONY: e2e-check e2e-live-create .e2e-live-install-cert-manager e2e-live-run e2e-live-faults e2e-live-update e2e-live-delete
 
 # Default target
 IMAGE_REPO ?= rustfs/operator
@@ -40,8 +40,9 @@ help:
 	@echo "  make console-fmt     - Format frontend code with Prettier (console-web)"
 	@echo "  make console-fmt-check - Check frontend formatting with Prettier (console-web)"
 	@echo "  make e2e-check        - Check Rust-native e2e harness (fmt + test + clippy)"
-	@echo "  make e2e-live-create  - Clean dedicated storage, recreate live Kind environment and load e2e image"
-	@echo "  make e2e-live-run     - Run all live suites (smoke/operator/console) in the existing live environment"
+	@echo "  make e2e-live-create  - Clean dedicated storage, recreate live Kind environment, install cert-manager, and load e2e image"
+	@echo "  make e2e-live-run     - Run all non-destructive live suites in the existing live environment"
+	@echo "  make e2e-live-faults  - Run destructive live fault suites with RUSTFS_E2E_DESTRUCTIVE=1"
 	@echo "  make e2e-live-update  - Rebuild image and update the live environment (load + rollout)"
 	@echo "  make e2e-live-delete  - Delete live Kind environment and clean dedicated storage"
 
@@ -85,6 +86,11 @@ build:
 E2E_MANIFEST ?= e2e/Cargo.toml
 E2E_BIN ?= cargo run --manifest-path $(E2E_MANIFEST) --bin rustfs-e2e --
 E2E_TEST_THREADS ?= 1
+E2E_KUBE_CONTEXT ?= kind-rustfs-e2e
+CERT_MANAGER_VERSION ?= v1.16.2
+CERT_MANAGER_MANIFEST_URL ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
+CERT_MANAGER_ROLLOUT_TIMEOUT ?= 180s
+E2E_LIVE_ENV ?= RUSTFS_E2E_LIVE=1 RUSTFS_E2E_CERT_MANAGER_VERSION=$(CERT_MANAGER_VERSION)
 
 # Rust-native e2e harness checks (non-live; ignored live tests remain opt-in)
 e2e-check:
@@ -96,27 +102,41 @@ e2e-check:
 e2e-live-create:
 	docker build --network host -t rustfs/operator:e2e .
 	docker build --network host -t rustfs/console-web:e2e -f console-web/Dockerfile console-web
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-delete || true
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-create
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-load-images
+	$(E2E_LIVE_ENV) $(E2E_BIN) kind-delete || true
+	$(E2E_LIVE_ENV) $(E2E_BIN) kind-create
+	$(E2E_LIVE_ENV) $(E2E_BIN) kind-load-images
+	$(MAKE) .e2e-live-install-cert-manager
+
+.e2e-live-install-cert-manager:
+	kubectl --context $(E2E_KUBE_CONTEXT) apply -f $(CERT_MANAGER_MANIFEST_URL)
+	kubectl --context $(E2E_KUBE_CONTEXT) -n cert-manager rollout status deployment/cert-manager --timeout=$(CERT_MANAGER_ROLLOUT_TIMEOUT)
+	kubectl --context $(E2E_KUBE_CONTEXT) -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=$(CERT_MANAGER_ROLLOUT_TIMEOUT)
+	kubectl --context $(E2E_KUBE_CONTEXT) -n cert-manager rollout status deployment/cert-manager-webhook --timeout=$(CERT_MANAGER_ROLLOUT_TIMEOUT)
 
 e2e-live-run:
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) assert-context
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) deploy-dev
+	$(E2E_LIVE_ENV) $(E2E_BIN) assert-context
+	$(MAKE) .e2e-live-install-cert-manager
+	$(E2E_LIVE_ENV) $(E2E_BIN) deploy-dev
+	$(E2E_LIVE_ENV) $(E2E_BIN) reset-live-fixtures
 	RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test smoke -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture
 	RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test operator -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture
+	RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test sts_functional -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture
 	RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test console -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture
+	RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test cert_manager_tls -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture
 	@echo "configured live e2e suites passed."
 
+e2e-live-faults:
+	RUSTFS_E2E_LIVE=1 RUSTFS_E2E_DESTRUCTIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test faults -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture
+
 e2e-live-update:
 	docker build --network host -t rustfs/operator:e2e .
 	docker build --network host -t rustfs/console-web:e2e -f console-web/Dockerfile console-web
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) assert-context
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-load-images
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) rollout-dev
+	$(E2E_LIVE_ENV) $(E2E_BIN) assert-context
+	$(E2E_LIVE_ENV) $(E2E_BIN) kind-load-images
+	$(E2E_LIVE_ENV) $(E2E_BIN) rollout-dev
 
 e2e-live-delete:
-	RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-delete
+	$(E2E_LIVE_ENV) $(E2E_BIN) kind-delete
 
 
 # Build Docker images. The operator image includes the controller and Console API;

README.md

@@ -68,8 +68,9 @@ From the repo root:
 | `make fmt` / `make clippy` / `make test` | Individual Rust checks. |
 | `make console-lint` / `make console-fmt-check` | Frontend only. |
 | `make e2e-check` | Validate the e2e harness without creating a live cluster. |
-| `make e2e-live-create` | Build e2e images, recreate the dedicated Kind cluster, and load images. |
-| `make e2e-live-run` | Deploy the dev control plane and run live smoke/operator/console suites. |
+| `make e2e-live-create` | Build e2e images, recreate the dedicated Kind cluster, install cert-manager, and load images. |
+| `make e2e-live-run` | Deploy the dev control plane and run all non-destructive live suites. |
+| `make e2e-live-faults` | Run destructive live fault suites with `RUSTFS_E2E_DESTRUCTIVE=1`. |
 | `make e2e-live-update` | Rebuild images, reload them into Kind, and roll out control-plane deployments. |
 | `make e2e-live-delete` | Delete the dedicated Kind cluster and its local storage. |
 

deploy/k8s-dev/operator-deployment.yaml

@@ -8,22 +8,45 @@ metadata:
   namespace: rustfs-system
   labels:
     app.kubernetes.io/name: rustfs-operator
+    app.kubernetes.io/component: operator
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: rustfs-operator
+      app.kubernetes.io/component: operator
   template:
     metadata:
       labels:
         app.kubernetes.io/name: rustfs-operator
+        app.kubernetes.io/component: operator
     spec:
       serviceAccountName: rustfs-operator
       containers:
         - name: operator
           image: rustfs/operator:dev
           imagePullPolicy: Never
           command: ["./operator", "server"]
+          ports:
+            - name: sts
+              containerPort: 4223
+              protocol: TCP
           env:
             - name: RUST_LOG
               value: info
+            - name: OPERATOR_STS_ENABLED
+              value: "true"
+            - name: OPERATOR_STS_AUDIENCE
+              value: sts.rustfs.com
+            - name: OPERATOR_STS_PORT
+              value: "4223"
+            - name: OPERATOR_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: OPERATOR_STS_SERVICE_NAME
+              value: rustfs-operator-sts
+            - name: OPERATOR_STS_TLS_ENABLED
+              value: "true"
+            - name: OPERATOR_STS_TLS_AUTO
+              value: "true"

deploy/k8s-dev/operator-rbac.yaml

@@ -33,6 +33,14 @@ rules:
   - apiGroups: ["rbac.authorization.k8s.io"]
     resources: ["roles", "rolebindings"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+  # STS/PolicyBinding authorization flow (PolicyBinding policy selection)
+  - apiGroups: ["sts.rustfs.com"]
+    resources: ["policybindings"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["authentication.k8s.io"]
+    resources: ["tokenreviews"]
+    verbs: ["create"]
   - apiGroups: ["apps"]
     resources: ["statefulsets"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

deploy/k8s-dev/operator-sts-service.yaml

@@ -0,0 +1,20 @@
+# Copyright 2025 RustFS Team
+# Operator STS Service (used by dev/e2e deployment flows)
+apiVersion: v1
+kind: Service
+metadata:
+  name: rustfs-operator-sts
+  namespace: rustfs-system
+  labels:
+    app.kubernetes.io/name: rustfs-operator
+    app.kubernetes.io/component: operator
+spec:
+  type: ClusterIP
+  ports:
+    - port: 4223
+      targetPort: sts
+      protocol: TCP
+      name: sts
+  selector:
+    app.kubernetes.io/name: rustfs-operator
+    app.kubernetes.io/component: operator

deploy/k8s-dev/policybinding-crd.yaml

@@ -0,0 +1,74 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: policybindings.sts.rustfs.com
+spec:
+  group: sts.rustfs.com
+  names:
+    categories: []
+    kind: PolicyBinding
+    plural: policybindings
+    shortNames:
+    - policybinding
+    singular: policybinding
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .status.currentState
+      name: State
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Auto-generated derived type for PolicyBindingSpec via `CustomResource`
+        properties:
+          spec:
+            properties:
+              application:
+                properties:
+                  namespace:
+                    type: string
+                  serviceaccount:
+                    type: string
+                required:
+                - namespace
+                - serviceaccount
+                type: object
+              policies:
+                items:
+                  type: string
+                type: array
+                x-kubernetes-validations:
+                - message: policies must contain at least one policy
+                  rule: self.size() > 0
+            required:
+            - application
+            - policies
+            type: object
+          status:
+            nullable: true
+            properties:
+              currentState:
+                nullable: true
+                type: string
+              usage:
+                nullable: true
+                properties:
+                  authorizations:
+                    format: uint64
+                    minimum: 0.0
+                    nullable: true
+                    type: integer
+                type: object
+            type: object
+        required:
+        - spec
+        title: PolicyBinding
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}