diff --git a/Cargo.lock b/Cargo.lock index d4d4499..cca316b 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.7.9" @@ -286,9 +308,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.44" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -302,6 +324,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.42" @@ -356,6 +384,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -547,6 +584,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -560,6 +598,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -639,9 +683,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" @@ -674,6 +718,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -793,9 +843,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] [[package]] @@ -823,6 +875,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -846,6 +917,21 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.12" @@ -922,6 +1008,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -950,6 +1037,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -971,6 +1059,7 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ + "base64", "bytes", "futures-channel", "futures-core", @@ -978,7 +1067,9 @@ dependencies = [ "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1130,6 +1221,22 @@ dependencies = [ "serde_core", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_debug" version = "1.1.0" @@ -1397,6 +1504,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.2.0" @@ -1529,15 +1642,22 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" name = "operator" version = "0.1.0" dependencies = [ + "async-trait", "axum", "chrono", "clap", "const-str", "futures", + "hex", + "hmac", "http", + "hyper", + "hyper-util", "jsonwebtoken", "k8s-openapi", "kube", + "rcgen", + "reqwest", "rustls", "rustls-pemfile", "rustls-webpki", @@ -1550,12 +1670,14 @@ dependencies = [ "snafu", "strum", "tokio", + "tokio-rustls", "tokio-stream", "tokio-util", "tower", "tower-http", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-swagger-ui", ] @@ -1710,6 +1832,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -1719,6 +1850,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.41" @@ -1734,6 +1920,48 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1792,6 +2020,44 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "reqwest" +version = "0.12.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + [[package]] name = "ring" version = "0.17.14" @@ -1840,12 +2106,19 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustls" version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -1882,6 +2155,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ + "web-time", "zeroize", ] @@ -1891,6 +2165,7 @@ version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2268,6 +2543,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -2372,6 +2650,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.49.0" @@ -2464,12 +2757,15 @@ dependencies = [ "bitflags", "bytes", "futures-core", + "futures-util", "http", "http-body", + "iri-string", "mime", "pin-project-lite", "tokio", "tokio-util", + "tower", "tower-layer", "tower-service", "tracing", @@ -2760,6 +3056,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.105" @@ -2792,6 +3101,35 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi-util" version = "0.1.11" @@ -3034,6 +3372,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index c3700df..889b3eb 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ chrono = { version = "0.4", features = ["serde"] } const-str = "1.0.0" serde = { version = "1.0.228", features = ["derive"] } tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "io-std", "io-util"] } +tokio-rustls = "0.26" tokio-stream = { version = "0.1", features = ["sync"] } tokio-util = { version = "0.7", features = ["io", "compat"] } futures = "0.3.31" @@ -27,18 +28,26 @@ clap = { version = "4.5.54", features = ["derive"] } rustls = { version = "0.23", default-features = false, features = ["ring"] } rustls-pemfile = "2.2.0" webpki = { package = "rustls-webpki", version = "0.103" } +rcgen = "0.13" sha2 = "0.10" +hmac = "0.12" +hex = "0.4" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +url = "2.5" shadow-rs = "1.5.0" snafu = { version = "0.8.9", features = ["futures"] } # Console dependencies axum = { version = "0.7", features = ["macros", "json"] } +hyper = "1" +hyper-util = { version = "0.1", features = ["server-auto", "service", "tokio"] } tower = "0.5" tower-http = { version = "0.6", features = ["cors", "trace", "compression-gzip"] } jsonwebtoken = "9.3" http = "1.2" utoipa = { version = "5", features = ["chrono"] } utoipa-swagger-ui = { version = "8", features = ["axum", "vendored"] } +async-trait = { version = "0.1.89", default-features = false } [dev-dependencies] @@ -50,4 +59,4 @@ unused_variables = "allow" [lints.clippy] unwrap_used = "deny" -expect_used = "deny" \ No newline at end of file +expect_used = "deny" diff --git a/Makefile b/Makefile index c17f750..b13c963 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ .PHONY: pre-commit fmt fmt-check clippy test build help .PHONY: docker-build-operator docker-build-console-web docker-build-all .PHONY: console-lint console-fmt console-fmt-check -.PHONY: e2e-check e2e-live-create e2e-live-run e2e-live-update e2e-live-delete +.PHONY: e2e-check e2e-live-create .e2e-live-install-cert-manager e2e-live-run e2e-live-faults e2e-live-update e2e-live-delete # Default target IMAGE_REPO ?= rustfs/operator @@ -40,8 +40,9 @@ help: @echo " make console-fmt - Format frontend code with Prettier (console-web)" @echo " make console-fmt-check - Check frontend formatting with Prettier (console-web)" @echo " make e2e-check - Check Rust-native e2e harness (fmt + test + clippy)" - @echo " make e2e-live-create - Clean dedicated storage, recreate live Kind environment and load e2e image" - @echo " make e2e-live-run - Run all live suites (smoke/operator/console) in the existing live environment" + @echo " make e2e-live-create - Clean dedicated storage, recreate live Kind environment, install cert-manager, and load e2e image" + @echo " make e2e-live-run - Run all non-destructive live suites in the existing live environment" + @echo " make e2e-live-faults - Run destructive live fault suites with RUSTFS_E2E_DESTRUCTIVE=1" @echo " make e2e-live-update - Rebuild image and update the live environment (load + rollout)" @echo " make e2e-live-delete - Delete live Kind environment and clean dedicated storage" @@ -85,6 +86,11 @@ build: E2E_MANIFEST ?= e2e/Cargo.toml E2E_BIN ?= cargo run --manifest-path $(E2E_MANIFEST) --bin rustfs-e2e -- E2E_TEST_THREADS ?= 1 +E2E_KUBE_CONTEXT ?= kind-rustfs-e2e +CERT_MANAGER_VERSION ?= v1.16.2 +CERT_MANAGER_MANIFEST_URL ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml +CERT_MANAGER_ROLLOUT_TIMEOUT ?= 180s +E2E_LIVE_ENV ?= RUSTFS_E2E_LIVE=1 RUSTFS_E2E_CERT_MANAGER_VERSION=$(CERT_MANAGER_VERSION) # Rust-native e2e harness checks (non-live; ignored live tests remain opt-in) e2e-check: @@ -96,27 +102,41 @@ e2e-check: e2e-live-create: docker build --network host -t rustfs/operator:e2e . docker build --network host -t rustfs/console-web:e2e -f console-web/Dockerfile console-web - RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-delete || true - RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-create - RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-load-images + $(E2E_LIVE_ENV) $(E2E_BIN) kind-delete || true + $(E2E_LIVE_ENV) $(E2E_BIN) kind-create + $(E2E_LIVE_ENV) $(E2E_BIN) kind-load-images + $(MAKE) .e2e-live-install-cert-manager + +.e2e-live-install-cert-manager: + kubectl --context $(E2E_KUBE_CONTEXT) apply -f $(CERT_MANAGER_MANIFEST_URL) + kubectl --context $(E2E_KUBE_CONTEXT) -n cert-manager rollout status deployment/cert-manager --timeout=$(CERT_MANAGER_ROLLOUT_TIMEOUT) + kubectl --context $(E2E_KUBE_CONTEXT) -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=$(CERT_MANAGER_ROLLOUT_TIMEOUT) + kubectl --context $(E2E_KUBE_CONTEXT) -n cert-manager rollout status deployment/cert-manager-webhook --timeout=$(CERT_MANAGER_ROLLOUT_TIMEOUT) e2e-live-run: - RUSTFS_E2E_LIVE=1 $(E2E_BIN) assert-context - RUSTFS_E2E_LIVE=1 $(E2E_BIN) deploy-dev + $(E2E_LIVE_ENV) $(E2E_BIN) assert-context + $(MAKE) .e2e-live-install-cert-manager + $(E2E_LIVE_ENV) $(E2E_BIN) deploy-dev + $(E2E_LIVE_ENV) $(E2E_BIN) reset-live-fixtures RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test smoke -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test operator -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture + RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test sts_functional -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test console -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture + RUSTFS_E2E_LIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test cert_manager_tls -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture @echo "configured live e2e suites passed." +e2e-live-faults: + RUSTFS_E2E_LIVE=1 RUSTFS_E2E_DESTRUCTIVE=1 cargo test --manifest-path $(E2E_MANIFEST) --test faults -- --ignored --test-threads=$(E2E_TEST_THREADS) --nocapture + e2e-live-update: docker build --network host -t rustfs/operator:e2e . docker build --network host -t rustfs/console-web:e2e -f console-web/Dockerfile console-web - RUSTFS_E2E_LIVE=1 $(E2E_BIN) assert-context - RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-load-images - RUSTFS_E2E_LIVE=1 $(E2E_BIN) rollout-dev + $(E2E_LIVE_ENV) $(E2E_BIN) assert-context + $(E2E_LIVE_ENV) $(E2E_BIN) kind-load-images + $(E2E_LIVE_ENV) $(E2E_BIN) rollout-dev e2e-live-delete: - RUSTFS_E2E_LIVE=1 $(E2E_BIN) kind-delete + $(E2E_LIVE_ENV) $(E2E_BIN) kind-delete # Build Docker images. The operator image includes the controller and Console API; diff --git a/README.md b/README.md index 3887090..5d1ae9b 100755 --- a/README.md +++ b/README.md @@ -68,8 +68,9 @@ From the repo root: | `make fmt` / `make clippy` / `make test` | Individual Rust checks. | | `make console-lint` / `make console-fmt-check` | Frontend only. | | `make e2e-check` | Validate the e2e harness without creating a live cluster. | -| `make e2e-live-create` | Build e2e images, recreate the dedicated Kind cluster, and load images. | -| `make e2e-live-run` | Deploy the dev control plane and run live smoke/operator/console suites. | +| `make e2e-live-create` | Build e2e images, recreate the dedicated Kind cluster, install cert-manager, and load images. | +| `make e2e-live-run` | Deploy the dev control plane and run all non-destructive live suites. | +| `make e2e-live-faults` | Run destructive live fault suites with `RUSTFS_E2E_DESTRUCTIVE=1`. | | `make e2e-live-update` | Rebuild images, reload them into Kind, and roll out control-plane deployments. | | `make e2e-live-delete` | Delete the dedicated Kind cluster and its local storage. | diff --git a/deploy/k8s-dev/operator-deployment.yaml b/deploy/k8s-dev/operator-deployment.yaml index 50b4842..3179078 100755 --- a/deploy/k8s-dev/operator-deployment.yaml +++ b/deploy/k8s-dev/operator-deployment.yaml @@ -8,15 +8,18 @@ metadata: namespace: rustfs-system labels: app.kubernetes.io/name: rustfs-operator + app.kubernetes.io/component: operator spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: rustfs-operator + app.kubernetes.io/component: operator template: metadata: labels: app.kubernetes.io/name: rustfs-operator + app.kubernetes.io/component: operator spec: serviceAccountName: rustfs-operator containers: @@ -24,6 +27,26 @@ spec: image: rustfs/operator:dev imagePullPolicy: Never command: ["./operator", "server"] + ports: + - name: sts + containerPort: 4223 + protocol: TCP env: - name: RUST_LOG value: info + - name: OPERATOR_STS_ENABLED + value: "true" + - name: OPERATOR_STS_AUDIENCE + value: sts.rustfs.com + - name: OPERATOR_STS_PORT + value: "4223" + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_STS_SERVICE_NAME + value: rustfs-operator-sts + - name: OPERATOR_STS_TLS_ENABLED + value: "true" + - name: OPERATOR_STS_TLS_AUTO + value: "true" diff --git a/deploy/k8s-dev/operator-rbac.yaml b/deploy/k8s-dev/operator-rbac.yaml index ae354d8..1625ebc 100755 --- a/deploy/k8s-dev/operator-rbac.yaml +++ b/deploy/k8s-dev/operator-rbac.yaml @@ -33,6 +33,14 @@ rules: - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + + # STS/PolicyBinding authorization flow (PolicyBinding policy selection) + - apiGroups: ["sts.rustfs.com"] + resources: ["policybindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] - apiGroups: ["apps"] resources: ["statefulsets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/deploy/k8s-dev/operator-sts-service.yaml b/deploy/k8s-dev/operator-sts-service.yaml new file mode 100644 index 0000000..446a8a0 --- /dev/null +++ b/deploy/k8s-dev/operator-sts-service.yaml @@ -0,0 +1,20 @@ +# Copyright 2025 RustFS Team +# Operator STS Service (used by dev/e2e deployment flows) +apiVersion: v1 +kind: Service +metadata: + name: rustfs-operator-sts + namespace: rustfs-system + labels: + app.kubernetes.io/name: rustfs-operator + app.kubernetes.io/component: operator +spec: + type: ClusterIP + ports: + - port: 4223 + targetPort: sts + protocol: TCP + name: sts + selector: + app.kubernetes.io/name: rustfs-operator + app.kubernetes.io/component: operator diff --git a/deploy/k8s-dev/policybinding-crd.yaml b/deploy/k8s-dev/policybinding-crd.yaml new file mode 100644 index 0000000..bbb5bd6 --- /dev/null +++ b/deploy/k8s-dev/policybinding-crd.yaml @@ -0,0 +1,74 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: policybindings.sts.rustfs.com +spec: + group: sts.rustfs.com + names: + categories: [] + kind: PolicyBinding + plural: policybindings + shortNames: + - policybinding + singular: policybinding + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.currentState + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for PolicyBindingSpec via `CustomResource` + properties: + spec: + properties: + application: + properties: + namespace: + type: string + serviceaccount: + type: string + required: + - namespace + - serviceaccount + type: object + policies: + items: + type: string + type: array + x-kubernetes-validations: + - message: policies must contain at least one policy + rule: self.size() > 0 + required: + - application + - policies + type: object + status: + nullable: true + properties: + currentState: + nullable: true + type: string + usage: + nullable: true + properties: + authorizations: + format: uint64 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - spec + title: PolicyBinding + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index 53db63e..6a0a46f 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -51,6 +51,32 @@ The following table lists the configurable parameters of the RustFS Operator cha | `operator.tolerations` | Tolerations for pod scheduling | `[]` | | `operator.affinity` | Affinity rules for pod scheduling | `{}` | +### Operator STS Configuration + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `sts.enabled` | Enable the operator STS endpoint | `true` | +| `sts.audience` | Kubernetes TokenReview audience expected by the operator STS endpoint | `sts.rustfs.com` | +| `sts.port` | Operator container port for STS | `4223` | +| `sts.tls.enabled` | Serve the operator STS endpoint over TLS | `true` | +| `sts.tls.auto` | Create the operator STS TLS Secret when missing | `true` | +| `sts.service.type` | Kubernetes Service type for STS | `ClusterIP` | +| `sts.service.port` | Kubernetes Service port for STS | `4223` | + +The RustFS operator STS endpoint intentionally uses an explicit Tenant route: + +```text +POST /sts/{tenantNamespace}/{tenantName} +``` + +This differs from MinIO Operator's namespace-only route. A `PolicyBinding` still lives in the Tenant namespace, but the workload must call STS with both the Tenant namespace and the Tenant name. + +The STS service is HTTPS by default. When `sts.tls.auto=true`, the operator creates the fixed `sts-tls` Secret in the operator namespace with `tls.crt`, `tls.key`, and `ca.crt`. Workloads must trust that CA. To use an externally issued certificate, pre-create `sts-tls` with a certificate signed by a CA already trusted by the workload and set `sts.tls.auto=false`. + +STS only issues credentials for TLS-enabled Tenants. For Tenant upstream calls, the operator selects the Tenant HTTPS service endpoint and trusts the CA recorded in `status.certificates.tls.caSecretRef`. + +Operator STS does not present a client certificate when calling the Tenant. Tenants configured with `spec.tls.certManager.caTrust.clientCaSecretRef` continue to run with server-side mTLS enabled, but Operator STS rejects those Tenants with HTTP 400 and `TenantTlsClientCertificateUnsupported`. + ### RBAC Configuration | Parameter | Description | Default | @@ -116,6 +142,81 @@ Install with your custom values: helm install rustfs-operator deploy/rustfs-operator/ -f custom-values.yaml ``` +### STS PolicyBinding and Workload Token + +Create a `PolicyBinding` in the target Tenant namespace. The binding authorizes one workload ServiceAccount to request temporary credentials for policies already defined in RustFS: + +```yaml +apiVersion: sts.rustfs.com/v1alpha1 +kind: PolicyBinding +metadata: + name: reports-readonly + namespace: storage +spec: + application: + namespace: reports + serviceaccount: reports-api + policies: + - readonly +``` + +The workload should mount a projected ServiceAccount token with an audience matching `sts.audience`: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: reports-api + namespace: reports +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reports-api + namespace: reports +spec: + replicas: 1 + selector: + matchLabels: + app: reports-api + template: + metadata: + labels: + app: reports-api + spec: + serviceAccountName: reports-api + containers: + - name: app + image: example/reports-api:latest + volumeMounts: + - name: rustfs-sts-token + mountPath: /var/run/secrets/rustfs-sts + readOnly: true + volumes: + - name: rustfs-sts-token + projected: + sources: + - serviceAccountToken: + path: token + audience: sts.rustfs.com + expirationSeconds: 3600 +``` + +The workload then calls the operator STS service with the target Tenant namespace and Tenant name: + +```bash +TOKEN="$(cat /var/run/secrets/rustfs-sts/token)" + +curl -sS -X POST \ + --cacert /var/run/secrets/rustfs-sts-ca/ca.crt \ + "https://rustfs-operator-sts.rustfs-system.svc:4223/sts/storage/rustfs-a" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "Version=2011-06-15" \ + --data-urlencode "Action=AssumeRoleWithWebIdentity" \ + --data-urlencode "WebIdentityToken=${TOKEN}" \ + --data-urlencode "DurationSeconds=3600" +``` + ## Creating Tenant Resources After installing the operator, you can create Tenant resources. See the project root `examples/` directory for sample manifests: diff --git a/deploy/rustfs-operator/crds/policybinding-crd.yaml b/deploy/rustfs-operator/crds/policybinding-crd.yaml new file mode 100644 index 0000000..bbb5bd6 --- /dev/null +++ b/deploy/rustfs-operator/crds/policybinding-crd.yaml @@ -0,0 +1,74 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: policybindings.sts.rustfs.com +spec: + group: sts.rustfs.com + names: + categories: [] + kind: PolicyBinding + plural: policybindings + shortNames: + - policybinding + singular: policybinding + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.currentState + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for PolicyBindingSpec via `CustomResource` + properties: + spec: + properties: + application: + properties: + namespace: + type: string + serviceaccount: + type: string + required: + - namespace + - serviceaccount + type: object + policies: + items: + type: string + type: array + x-kubernetes-validations: + - message: policies must contain at least one policy + rule: self.size() > 0 + required: + - application + - policies + type: object + status: + nullable: true + properties: + currentState: + nullable: true + type: string + usage: + nullable: true + properties: + authorizations: + format: uint64 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - spec + title: PolicyBinding + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/rustfs-operator/templates/clusterrole.yaml b/deploy/rustfs-operator/templates/clusterrole.yaml index 02be419..af36de6 100755 --- a/deploy/rustfs-operator/templates/clusterrole.yaml +++ b/deploy/rustfs-operator/templates/clusterrole.yaml @@ -32,6 +32,14 @@ rules: resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # STS / PolicyBinding authorization flow + - apiGroups: ["sts.rustfs.com"] + resources: ["policybindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + # StatefulSets for tenant pools - apiGroups: ["apps"] resources: ["statefulsets"] diff --git a/deploy/rustfs-operator/templates/deployment.yaml b/deploy/rustfs-operator/templates/deployment.yaml index 0519fa5..18fffe3 100755 --- a/deploy/rustfs-operator/templates/deployment.yaml +++ b/deploy/rustfs-operator/templates/deployment.yaml @@ -5,6 +5,7 @@ metadata: namespace: {{ include "rustfs-operator.namespace" . }} labels: {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: operator {{- with .Values.commonAnnotations }} annotations: {{- toYaml . | nindent 4 }} @@ -14,10 +15,12 @@ spec: selector: matchLabels: {{- include "rustfs-operator.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: operator template: metadata: labels: {{- include "rustfs-operator.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: operator spec: serviceAccountName: {{ include "rustfs-operator.serviceAccountName" . }} {{- with .Values.operator.imagePullSecrets }} @@ -33,8 +36,32 @@ spec: image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" imagePullPolicy: {{ .Values.operator.image.pullPolicy }} command: ["./operator", "server"] - {{- with .Values.operator.env }} + {{- if .Values.sts.enabled }} + ports: + - name: sts + containerPort: {{ .Values.sts.port }} + protocol: TCP + {{- end }} env: + - name: OPERATOR_STS_ENABLED + value: {{ .Values.sts.enabled | quote }} + - name: OPERATOR_STS_AUDIENCE + value: {{ .Values.sts.audience | quote }} + {{- if .Values.sts.enabled }} + - name: OPERATOR_STS_PORT + value: {{ .Values.sts.port | quote }} + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_STS_SERVICE_NAME + value: {{ printf "%s-sts" (include "rustfs-operator.fullname" .) | quote }} + - name: OPERATOR_STS_TLS_ENABLED + value: {{ .Values.sts.tls.enabled | quote }} + - name: OPERATOR_STS_TLS_AUTO + value: {{ .Values.sts.tls.auto | quote }} + {{- end }} + {{- with .Values.operator.env }} {{- toYaml . | nindent 12 }} {{- end }} {{- with .Values.operator.resources }} diff --git a/deploy/rustfs-operator/templates/operator-sts-service.yaml b/deploy/rustfs-operator/templates/operator-sts-service.yaml new file mode 100644 index 0000000..df4f953 --- /dev/null +++ b/deploy/rustfs-operator/templates/operator-sts-service.yaml @@ -0,0 +1,30 @@ +{{- if .Values.sts.enabled -}} +{{- if ne .Values.sts.service.type "ClusterIP" -}} +{{- fail "operator STS currently supports only ClusterIP; expose it outside the cluster through an explicitly configured ingress or gateway" -}} +{{- end -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "rustfs-operator.fullname" . }}-sts + namespace: {{ include "rustfs-operator.namespace" . }} + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: operator + {{- with .Values.sts.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.sts.service.type }} + {{- if and (eq .Values.sts.service.type "ClusterIP") .Values.sts.service.clusterIP }} + clusterIP: {{ .Values.sts.service.clusterIP }} + {{- end }} + ports: + - port: {{ .Values.sts.service.port }} + targetPort: sts + protocol: TCP + name: sts + selector: + {{- include "rustfs-operator.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: operator +{{- end }} diff --git a/deploy/rustfs-operator/values.yaml b/deploy/rustfs-operator/values.yaml index c775bb9..5644fa4 100755 --- a/deploy/rustfs-operator/values.yaml +++ b/deploy/rustfs-operator/values.yaml @@ -51,6 +51,26 @@ operator: # Affinity rules for operator pod scheduling affinity: {} +# Kubernetes ServiceAccount for operator STS endpoint +sts: + # Enable/disable operator STS exposure + enabled: true + # Kubernetes TokenReview audience used by RustFS STS clients. + # Workloads should request projected service account tokens for this audience. + audience: sts.rustfs.com + # STS service/port in operator pods + port: 4223 + tls: + # Operator STS listens with TLS by default and uses this Secret for server certs. + enabled: true + # When true, the operator creates the Secret if it is missing. + auto: true + service: + type: ClusterIP + port: 4223 + clusterIP: "" + annotations: {} + # ServiceAccount configuration serviceAccount: # Specifies whether a service account should be created diff --git a/e2e/Cargo.lock b/e2e/Cargo.lock index 785b508..b5456d6 100644 --- a/e2e/Cargo.lock +++ b/e2e/Cargo.lock @@ -179,6 +179,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.7.9" @@ -368,6 +390,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -590,6 +621,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -612,6 +644,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -736,6 +774,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -899,6 +943,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -922,6 +985,21 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.12" @@ -997,6 +1075,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1659,15 +1738,22 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" name = "operator" version = "0.1.0" dependencies = [ + "async-trait", "axum", "chrono", "clap", "const-str", "futures", + "hex", + "hmac", "http", + "hyper", + "hyper-util", "jsonwebtoken", "k8s-openapi", "kube", + "rcgen", + "reqwest", "rustls", "rustls-pemfile", "rustls-webpki", @@ -1680,12 +1766,14 @@ dependencies = [ "snafu", "strum", "tokio", + "tokio-rustls", "tokio-stream", "tokio-util", "tower", "tower-http", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-swagger-ui", ] @@ -1983,6 +2071,19 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2152,6 +2253,7 @@ name = "rustfs-operator-e2e" version = "0.1.0" dependencies = [ "anyhow", + "axum", "futures", "k8s-openapi", "kube", @@ -2162,6 +2264,7 @@ dependencies = [ "serde_yaml_ng", "tempfile", "tokio", + "tower", "uuid", ] @@ -2184,6 +2287,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -2230,6 +2334,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3582,6 +3687,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/e2e/Cargo.toml b/e2e/Cargo.toml index 8bd2fe5..b8a74fd 100644 --- a/e2e/Cargo.toml +++ b/e2e/Cargo.toml @@ -8,12 +8,14 @@ publish = false operator = { path = ".." } anyhow = "1" +axum = { version = "0.7", features = ["macros"] } futures = "0.3.31" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" serde_yaml_ng = "0.10.0" tempfile = "3" tokio = { version = "1.49.0", features = ["rt-multi-thread", "macros", "process", "time", "io-util"] } +tower = "0.5" uuid = { version = "1", features = ["v4"] } k8s-openapi = { version = "0.26.1", features = ["v1_30", "schemars"] } diff --git a/e2e/src/bin/rustfs-e2e.rs b/e2e/src/bin/rustfs-e2e.rs index a698a40..a3dc28b 100644 --- a/e2e/src/bin/rustfs-e2e.rs +++ b/e2e/src/bin/rustfs-e2e.rs @@ -14,7 +14,8 @@ use anyhow::{Result, bail}; use rustfs_operator_e2e::framework::{ - command::CommandSpec, config::E2eConfig, deploy, images::ImageSet, kind::KindCluster, live, + cert_manager_tls, command::CommandSpec, config::E2eConfig, deploy, images::ImageSet, + kind::KindCluster, live, resources, storage, }; fn main() -> Result<()> { @@ -28,6 +29,8 @@ fn main() -> Result<()> { "assert-context" => assert_context(&config), "kind-create" => create_kind_cluster(&config), "kind-delete" => delete_kind_cluster(&config), + "sanitize-live-storage" => sanitize_live_storage(&config), + "reset-live-fixtures" | "reset-live-smoke-fixture" => reset_live_fixtures(&config), "kind-load-images" => load_images(&config), "deploy-dev" => deploy_dev(&config), "rollout-dev" => rollout_dev(&config), @@ -48,7 +51,11 @@ fn print_help() -> Result<()> { println!(" assert-context Require RUSTFS_E2E_LIVE=1 and dedicated Kind context"); println!(" kind-create Create the dedicated Kind cluster"); println!(" kind-delete Delete the dedicated Kind cluster and storage"); - println!(" kind-load-images Load operator, console-web, and RustFS images into Kind"); + println!(" sanitize-live-storage"); + println!(" reset-live-fixtures"); + println!( + " kind-load-images Load operator, console-web, RustFS, and dependency images into Kind" + ); println!(" deploy-dev Apply operator/console manifests into dedicated Kind"); println!(" rollout-dev Restart and wait for e2e control-plane deployments"); Ok(()) @@ -77,6 +84,38 @@ fn delete_kind_cluster(config: &E2eConfig) -> Result<()> { Ok(()) } +fn sanitize_live_storage(config: &E2eConfig) -> Result<()> { + live::require_live_enabled(config)?; + let kind = KindCluster::new(config.clone()); + let stale_paths = kind.stale_local_rustfs_format_paths()?; + + if stale_paths.is_empty() { + println!("no stale rustfs format metadata found in dedicated host storage"); + return Ok(()); + } + + println!( + "detected {} stale rustfs format file(s) in dedicated host storage", + stale_paths.len() + ); + for path in stale_paths { + println!(" - {}", path.display()); + } + + bail!( + "refusing to reset live host storage while the Kind cluster may still be running; recreate the dedicated e2e cluster with `make e2e-live-create`" + ) +} + +fn reset_live_fixtures(config: &E2eConfig) -> Result<()> { + live::require_live_enabled(config)?; + live::ensure_dedicated_context(config)?; + resources::reset_smoke_tenant_resources(config)?; + storage::reset_default_local_storage(config)?; + cert_manager_tls::reset_positive_case_resources(config)?; + Ok(()) +} + fn load_images(config: &E2eConfig) -> Result<()> { live::require_live_enabled(config)?; @@ -92,8 +131,8 @@ fn load_images(config: &E2eConfig) -> Result<()> { let kind = KindCluster::new(config.clone()); for image in images.all() { - println!("loading {image} into {}", config.cluster_name); - kind.load_image_command(image).run_checked()?; + println!("loading {image} into {} nodes", config.cluster_name); + kind.load_image(image)?; } Ok(()) } diff --git a/e2e/src/cases/mod.rs b/e2e/src/cases/mod.rs index d227667..04933f3 100644 --- a/e2e/src/cases/mod.rs +++ b/e2e/src/cases/mod.rs @@ -16,12 +16,14 @@ pub mod cert_manager_tls; pub mod console; pub mod operator; pub mod smoke; +pub mod sts; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Suite { Smoke, Operator, Console, + Sts, CertManagerTls, } @@ -56,6 +58,7 @@ pub fn all_cases() -> Vec { let mut cases = Vec::new(); cases.extend(smoke::cases()); cases.extend(operator::cases()); + cases.extend(sts::cases()); cases.extend(console::cases()); cases.extend(cert_manager_tls::cases()); cases @@ -73,6 +76,7 @@ mod tests { assert!(suites.contains(&Suite::Smoke)); assert!(suites.contains(&Suite::Operator)); + assert!(suites.contains(&Suite::Sts)); assert!(suites.contains(&Suite::Console)); assert!(suites.contains(&Suite::CertManagerTls)); } @@ -114,6 +118,7 @@ mod tests { assert_eq!(counts.get(&Suite::Smoke).copied().unwrap_or_default(), 3); assert_eq!(counts.get(&Suite::Operator).copied().unwrap_or_default(), 1); + assert_eq!(counts.get(&Suite::Sts).copied().unwrap_or_default(), 2); assert_eq!(counts.get(&Suite::Console).copied().unwrap_or_default(), 1); assert_eq!( counts diff --git a/e2e/src/cases/sts.rs b/e2e/src/cases/sts.rs new file mode 100644 index 0000000..7f16711 --- /dev/null +++ b/e2e/src/cases/sts.rs @@ -0,0 +1,55 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{CaseSpec, Suite}; + +pub fn cases() -> Vec { + vec![ + CaseSpec::new( + Suite::Sts, + "sts_live_tokenreview_policybinding_and_assume_role_succeeds", + "Create a TLS Tenant, projected ServiceAccount token, PolicyBinding, and RustFS canned policy, then verify HTTPS Operator STS returns temporary S3 credentials.", + "sts/tls-tokenreview-policybinding-assumerole", + "sts", + ), + CaseSpec::new( + Suite::Sts, + "sts_live_rejects_non_tls_tenant", + "Verify HTTPS Operator STS rejects a non-TLS Tenant before issuing temporary credentials.", + "sts/non-tls-tenant-rejected", + "sts", + ), + ] +} + +#[cfg(test)] +mod tests { + use super::cases; + + #[test] + fn sts_case_inventory_matches_executable_tests() { + let names = cases() + .into_iter() + .map(|case| case.name) + .collect::>(); + + assert_eq!( + names, + vec![ + "sts_live_tokenreview_policybinding_and_assume_role_succeeds", + "sts_live_rejects_non_tls_tenant" + ] + ); + } +} diff --git a/e2e/src/framework/cert_manager_tls.rs b/e2e/src/framework/cert_manager_tls.rs index 39126c6..15caa66 100644 --- a/e2e/src/framework/cert_manager_tls.rs +++ b/e2e/src/framework/cert_manager_tls.rs @@ -99,6 +99,23 @@ pub fn external_secret_storage_layout(config: &E2eConfig) -> storage::LocalStora positive_tls_storage_layout(config, EXTERNAL_SECRET_CASE_SUFFIX) } +pub fn reset_positive_case_resources(config: &E2eConfig) -> Result<()> { + let managed = managed_certificate_case_config(config); + resources::reset_smoke_tenant_resources(&managed)?; + storage::reset_local_storage_for_layout( + &managed, + &managed_certificate_storage_layout(&managed), + ) + .context("reset managed cert-manager TLS e2e storage")?; + + let external = external_secret_case_config(config); + resources::reset_smoke_tenant_resources(&external)?; + storage::reset_local_storage_for_layout(&external, &external_secret_storage_layout(&external)) + .context("reset external Secret cert-manager TLS e2e storage")?; + + Ok(()) +} + pub fn managed_secret_name(config: &E2eConfig) -> String { format!("{}-managed-tls", config.tenant_name) } diff --git a/e2e/src/framework/config.rs b/e2e/src/framework/config.rs index de7af80..c3c4cfe 100644 --- a/e2e/src/framework/config.rs +++ b/e2e/src/framework/config.rs @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use operator::types::v1alpha1::k8s::PodManagementPolicy; use std::path::PathBuf; use std::time::Duration; pub const DEFAULT_CLUSTER_NAME: &str = "rustfs-e2e"; pub const DEFAULT_STORAGE_HOST_DIR_PREFIX: &str = "/tmp/rustfs-e2e-storage"; pub const DEFAULT_RUSTFS_IMAGE: &str = "rustfs/rustfs:latest"; +pub const DEFAULT_CERT_MANAGER_VERSION: &str = "v1.16.2"; pub const KIND_WORKER_COUNT: usize = 3; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct E2eConfig { pub cluster_name: String, pub context: String, @@ -33,6 +35,8 @@ pub struct E2eConfig { pub operator_image: String, pub console_web_image: String, pub rustfs_image: String, + pub cert_manager_version: String, + pub pod_management_policy: Option, pub kind_config: PathBuf, pub artifacts_dir: PathBuf, pub live_enabled: bool, @@ -71,6 +75,11 @@ impl E2eConfig { operator_image: "rustfs/operator:e2e".to_string(), console_web_image: "rustfs/console-web:e2e".to_string(), rustfs_image: env_or(&get_env, "RUSTFS_E2E_SERVER_IMAGE", DEFAULT_RUSTFS_IMAGE), + cert_manager_version: env_or( + &get_env, + "RUSTFS_E2E_CERT_MANAGER_VERSION", + DEFAULT_CERT_MANAGER_VERSION, + ), kind_config: PathBuf::from(env_or( &get_env, "RUSTFS_E2E_KIND_CONFIG", @@ -81,6 +90,7 @@ impl E2eConfig { "RUSTFS_E2E_ARTIFACTS", "target/e2e/artifacts", )), + pod_management_policy: parse_pod_management_policy(&get_env), live_enabled: env_bool(&get_env, "RUSTFS_E2E_LIVE"), destructive_enabled: env_bool(&get_env, "RUSTFS_E2E_DESTRUCTIVE"), timeout: Duration::from_secs(env_u64(&get_env, "RUSTFS_E2E_TIMEOUT_SECONDS", 300)), @@ -126,6 +136,20 @@ where .unwrap_or(default) } +fn parse_pod_management_policy(get_env: &F) -> Option +where + F: Fn(&str) -> Option, +{ + let raw = get_env("RUSTFS_E2E_POD_MANAGEMENT_POLICY")?; + match raw.to_ascii_lowercase().as_str() { + "parallel" => Some(PodManagementPolicy::Parallel), + "orderedready" | "ordered_ready" | "ordered-ready" => { + Some(PodManagementPolicy::OrderedReady) + } + _ => None, + } +} + #[cfg(test)] mod tests { use super::{DEFAULT_RUSTFS_IMAGE, E2eConfig}; @@ -141,6 +165,7 @@ mod tests { assert_eq!(config.storage_class, "local-storage"); assert_eq!(config.pv_count, 12); assert_eq!(config.rustfs_image, DEFAULT_RUSTFS_IMAGE); + assert_eq!(config.cert_manager_version, "v1.16.2"); assert_eq!( config.kind_config, std::path::PathBuf::from("e2e/manifests/kind-rustfs-e2e.yaml") @@ -157,6 +182,7 @@ mod tests { "RUSTFS_E2E_OPERATOR_IMAGE" => Some("rustfs/operator:other".to_string()), "RUSTFS_E2E_CONSOLE_WEB_IMAGE" => Some("rustfs/console-web:other".to_string()), "RUSTFS_E2E_SERVER_IMAGE" => Some("rustfs/rustfs:dev".to_string()), + "RUSTFS_E2E_CERT_MANAGER_VERSION" => Some("v9.9.9".to_string()), "RUSTFS_E2E_LIVE" => Some("true".to_string()), _ => None, }); @@ -166,6 +192,7 @@ mod tests { assert_eq!(config.operator_image, "rustfs/operator:e2e"); assert_eq!(config.console_web_image, "rustfs/console-web:e2e"); assert_eq!(config.rustfs_image, "rustfs/rustfs:dev"); + assert_eq!(config.cert_manager_version, "v9.9.9"); assert!(config.live_enabled); } } diff --git a/e2e/src/framework/deploy.rs b/e2e/src/framework/deploy.rs index b04e8f9..832f2c1 100644 --- a/e2e/src/framework/deploy.rs +++ b/e2e/src/framework/deploy.rs @@ -16,13 +16,16 @@ use anyhow::Result; use crate::framework::{config::E2eConfig, kubectl::Kubectl, resources}; -const CONTROL_PLANE_CRD: &str = - include_str!("../../../deploy/rustfs-operator/crds/tenant-crd.yaml"); +const TENANT_CRD: &str = include_str!("../../../deploy/rustfs-operator/crds/tenant-crd.yaml"); +const POLICY_BINDING_CRD: &str = + include_str!("../../../deploy/rustfs-operator/crds/policybinding-crd.yaml"); const OPERATOR_RBAC: &str = include_str!("../../../deploy/k8s-dev/operator-rbac.yaml"); const CONSOLE_RBAC: &str = include_str!("../../../deploy/k8s-dev/console-rbac.yaml"); const OPERATOR_DEPLOYMENT: &str = include_str!("../../../deploy/k8s-dev/operator-deployment.yaml"); const CONSOLE_DEPLOYMENT: &str = include_str!("../../../deploy/k8s-dev/console-deployment.yaml"); const CONSOLE_SERVICE: &str = include_str!("../../../deploy/k8s-dev/console-service.yaml"); +const OPERATOR_STS_SERVICE: &str = + include_str!("../../../deploy/k8s-dev/operator-sts-service.yaml"); const CONSOLE_FRONTEND_DEPLOYMENT: &str = include_str!("../../../deploy/k8s-dev/console-frontend-deployment.yaml"); const CONSOLE_FRONTEND_SERVICE: &str = @@ -45,8 +48,9 @@ pub fn deploy_dev(config: &E2eConfig) -> Result<()> { .apply_yaml_command(resources::namespace_manifest(&config.operator_namespace)) .run_checked()?; + kubectl.apply_yaml_command(TENANT_CRD).run_checked()?; kubectl - .apply_yaml_command(CONTROL_PLANE_CRD) + .apply_yaml_command(POLICY_BINDING_CRD) .run_checked()?; kubectl @@ -71,6 +75,9 @@ pub fn deploy_dev(config: &E2eConfig) -> Result<()> { )) .run_checked()?; kubectl.apply_yaml_command(CONSOLE_SERVICE).run_checked()?; + kubectl + .apply_yaml_command(OPERATOR_STS_SERVICE) + .run_checked()?; kubectl .apply_yaml_command(patch_images_and_tags( CONSOLE_FRONTEND_DEPLOYMENT, diff --git a/e2e/src/framework/images.rs b/e2e/src/framework/images.rs index 8e91c28..73c754a 100644 --- a/e2e/src/framework/images.rs +++ b/e2e/src/framework/images.rs @@ -19,19 +19,44 @@ pub struct ImageSet { pub operator: String, pub console_web: String, pub rustfs: String, + pub cert_manager_controller: String, + pub cert_manager_webhook: String, + pub cert_manager_cainjector: String, + pub cert_manager_acmesolver: String, } impl ImageSet { pub fn from_config(config: &E2eConfig) -> Self { + let cert_manager_version = &config.cert_manager_version; Self { operator: config.operator_image.clone(), console_web: config.console_web_image.clone(), rustfs: config.rustfs_image.clone(), + cert_manager_controller: format!( + "quay.io/jetstack/cert-manager-controller:{cert_manager_version}" + ), + cert_manager_webhook: format!( + "quay.io/jetstack/cert-manager-webhook:{cert_manager_version}" + ), + cert_manager_cainjector: format!( + "quay.io/jetstack/cert-manager-cainjector:{cert_manager_version}" + ), + cert_manager_acmesolver: format!( + "quay.io/jetstack/cert-manager-acmesolver:{cert_manager_version}" + ), } } - pub fn all(&self) -> [&str; 3] { - [&self.operator, &self.console_web, &self.rustfs] + pub fn all(&self) -> [&str; 7] { + [ + &self.operator, + &self.console_web, + &self.rustfs, + &self.cert_manager_controller, + &self.cert_manager_webhook, + &self.cert_manager_cainjector, + &self.cert_manager_acmesolver, + ] } } @@ -49,7 +74,11 @@ mod tests { [ "rustfs/operator:e2e", "rustfs/console-web:e2e", - "rustfs/rustfs:latest" + "rustfs/rustfs:latest", + "quay.io/jetstack/cert-manager-controller:v1.16.2", + "quay.io/jetstack/cert-manager-webhook:v1.16.2", + "quay.io/jetstack/cert-manager-cainjector:v1.16.2", + "quay.io/jetstack/cert-manager-acmesolver:v1.16.2" ] ); } diff --git a/e2e/src/framework/kind.rs b/e2e/src/framework/kind.rs index 9e1285b..3cb576d 100644 --- a/e2e/src/framework/kind.rs +++ b/e2e/src/framework/kind.rs @@ -22,6 +22,8 @@ use crate::framework::{ config::{DEFAULT_STORAGE_HOST_DIR_PREFIX, E2eConfig, KIND_WORKER_COUNT}, }; +const RUSTFS_FORMAT_MARKER_PATHS: [&str; 2] = [".rustfs.sys/format.json", ".minio.sys/format.json"]; + #[derive(Debug, Clone)] pub struct KindCluster { config: E2eConfig, @@ -73,6 +75,32 @@ impl KindCluster { Ok(()) } + pub fn stale_local_rustfs_format_paths(&self) -> Result> { + let mut stale_paths = Vec::new(); + + for dir in self.host_storage_dirs() { + if !dir.exists() { + continue; + } + + for entry in fs::read_dir(&dir)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + + for marker_path in RUSTFS_FORMAT_MARKER_PATHS { + let format_path = entry.path().join(marker_path); + if format_path.exists() { + stale_paths.push(format_path); + } + } + } + } + + Ok(stale_paths) + } + fn remove_host_storage_dir(&self, dir: &Path) -> Result<()> { ensure_existing_dedicated_host_storage_dir_is_safe(dir)?; println!("removing dedicated e2e storage dir {}", dir.display()); @@ -119,13 +147,47 @@ impl KindCluster { ]) } - pub fn load_image_command(&self, image: &str) -> CommandSpec { - CommandSpec::new("kind").args([ - "load".to_string(), - "docker-image".to_string(), + pub fn load_image(&self, image: &str) -> Result<()> { + for node in self.node_names()? { + println!("loading {image} into {node} through containerd import"); + self.ctr_import_image_command(image, &node) + .run_checked() + .with_context(|| format!("import {image} into Kind node {node}"))?; + } + Ok(()) + } + + fn node_names(&self) -> Result> { + let output = CommandSpec::new("kind") + .args([ + "get".to_string(), + "nodes".to_string(), + "--name".to_string(), + self.config.cluster_name.clone(), + ]) + .run_checked()?; + let nodes: Vec = output + .stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToString::to_string) + .collect(); + + if nodes.is_empty() { + bail!("kind cluster {} has no nodes", self.config.cluster_name); + } + + Ok(nodes) + } + + fn ctr_import_image_command(&self, image: &str, node: &str) -> CommandSpec { + CommandSpec::new("sh").args([ + "-c".to_string(), + "docker save \"$1\" | docker exec --privileged -i \"$2\" ctr --namespace=k8s.io images import --digests --snapshotter=overlayfs -".to_string(), + "sh".to_string(), image.to_string(), - "--name".to_string(), - self.config.cluster_name.clone(), + node.to_string(), ]) } } @@ -222,13 +284,13 @@ mod tests { use crate::framework::config::E2eConfig; #[test] - fn load_image_command_targets_the_dedicated_cluster() { + fn ctr_import_image_command_streams_docker_archive_to_node_containerd() { let kind = KindCluster::new(E2eConfig::defaults()); - let command = kind.load_image_command("rustfs/operator:e2e"); + let command = kind.ctr_import_image_command("rustfs/rustfs:latest", "rustfs-e2e-worker"); assert_eq!( command.display(), - "kind load docker-image rustfs/operator:e2e --name rustfs-e2e" + "sh -c docker save \"$1\" | docker exec --privileged -i \"$2\" ctr --namespace=k8s.io images import --digests --snapshotter=overlayfs - sh rustfs/rustfs:latest rustfs-e2e-worker" ); } diff --git a/e2e/src/framework/kube_client.rs b/e2e/src/framework/kube_client.rs index 881b542..adec679 100644 --- a/e2e/src/framework/kube_client.rs +++ b/e2e/src/framework/kube_client.rs @@ -17,6 +17,8 @@ use kube::{Api, Client}; use operator::types::v1alpha1::tenant::Tenant; pub async fn default_client() -> Result { + operator::install_rustls_crypto_provider(); + Ok(Client::try_default().await?) } diff --git a/e2e/src/framework/port_forward.rs b/e2e/src/framework/port_forward.rs index 347ea87..d751ec2 100644 --- a/e2e/src/framework/port_forward.rs +++ b/e2e/src/framework/port_forward.rs @@ -45,6 +45,24 @@ impl PortForwardSpec { } } + pub fn operator_sts(namespace: impl Into) -> Self { + Self { + namespace: namespace.into(), + service: "svc/rustfs-operator-sts".to_string(), + local_port: 14223, + remote_port: 4223, + } + } + + pub fn tenant_io(namespace: impl Into, tenant_name: impl Into) -> Self { + Self { + namespace: namespace.into(), + service: format!("svc/{}-io", tenant_name.into()), + local_port: 19000, + remote_port: 9000, + } + } + pub fn command(&self, kubectl: &Kubectl) -> CommandSpec { kubectl.clone().namespaced(&self.namespace).command([ "port-forward".to_string(), @@ -82,6 +100,16 @@ impl PortForwardSpec { let kubectl = Kubectl::new(config); Self::console(&config.operator_namespace).start_with_temp_log(&kubectl) } + + pub fn start_operator_sts(config: &E2eConfig) -> Result { + let kubectl = Kubectl::new(config); + Self::operator_sts(&config.operator_namespace).start_with_temp_log(&kubectl) + } + + pub fn start_tenant_io(config: &E2eConfig) -> Result { + let kubectl = Kubectl::new(config); + Self::tenant_io(&config.test_namespace, &config.tenant_name).start_with_temp_log(&kubectl) + } } impl PortForwardGuard { @@ -138,4 +166,27 @@ mod tests { "kubectl --context kind-rustfs-e2e -n rustfs-system port-forward svc/rustfs-operator-console 19090:9090" ); } + + #[test] + fn sts_port_forward_targets_operator_sts_service() { + let kubectl = Kubectl::new(&E2eConfig::defaults()); + let command = PortForwardSpec::operator_sts("rustfs-system").command(&kubectl); + + assert_eq!( + command.display(), + "kubectl --context kind-rustfs-e2e -n rustfs-system port-forward svc/rustfs-operator-sts 14223:4223" + ); + } + + #[test] + fn tenant_io_port_forward_targets_tenant_service() { + let kubectl = Kubectl::new(&E2eConfig::defaults()); + let command = + PortForwardSpec::tenant_io("rustfs-e2e-smoke", "e2e-tenant").command(&kubectl); + + assert_eq!( + command.display(), + "kubectl --context kind-rustfs-e2e -n rustfs-e2e-smoke port-forward svc/e2e-tenant-io 19000:9000" + ); + } } diff --git a/e2e/src/framework/resources.rs b/e2e/src/framework/resources.rs index c0e91bc..c6b36bb 100644 --- a/e2e/src/framework/resources.rs +++ b/e2e/src/framework/resources.rs @@ -12,12 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::Result; +use anyhow::{Context, Result, bail}; +use std::thread::sleep; +use std::time::{Duration, Instant}; -use crate::framework::{config::E2eConfig, kubectl::Kubectl, tenant_factory::TenantTemplate}; +use crate::framework::{ + command::{CommandOutput, CommandSpec}, + config::E2eConfig, + kubectl::Kubectl, + tenant_factory::TenantTemplate, +}; +use operator::types::v1alpha1::k8s::PodManagementPolicy; const E2E_ACCESS_KEY: &str = "e2eaccess"; const E2E_SECRET_KEY: &str = "e2esecret"; +const RESOURCE_RESET_TIMEOUT: Duration = Duration::from_secs(120); +const RESOURCE_RESET_POLL_INTERVAL: Duration = Duration::from_secs(2); pub fn credential_secret_name(config: &E2eConfig) -> String { format!("{}-credentials", config.tenant_name) @@ -53,13 +63,22 @@ stringData: } pub fn smoke_tenant_template(config: &E2eConfig) -> TenantTemplate { - TenantTemplate::kind_local( + let mut template = TenantTemplate::kind_local( &config.test_namespace, &config.tenant_name, &config.rustfs_image, &config.storage_class, credential_secret_name(config), - ) + ); + + template.pod_management_policy = Some( + config + .pod_management_policy + .clone() + .unwrap_or(PodManagementPolicy::Parallel), + ); + + template } pub fn smoke_tenant_manifest(config: &E2eConfig) -> Result { @@ -82,6 +101,203 @@ pub fn apply_smoke_tenant_resources(config: &E2eConfig) -> Result<()> { Ok(()) } +pub fn reset_and_apply_smoke_tenant_resources(config: &E2eConfig) -> Result<()> { + reset_smoke_tenant_resources(config)?; + apply_smoke_tenant_resources(config) +} + +pub fn reset_smoke_tenant_resources(config: &E2eConfig) -> Result<()> { + let kubectl = Kubectl::new(config); + if !namespace_exists(&kubectl, &config.test_namespace)? { + return Ok(()); + } + + let kubectl = kubectl.namespaced(&config.test_namespace); + let selector = format!("rustfs.tenant={}", config.tenant_name); + + run_delete(kubectl.command([ + "delete", + "tenant", + &config.tenant_name, + "--ignore-not-found", + "--wait=false", + ]))?; + run_delete(kubectl.command([ + "delete", + "statefulset", + "-l", + &selector, + "--ignore-not-found", + "--wait=false", + ]))?; + run_delete(kubectl.command([ + "delete", + "pod", + "-l", + &selector, + "--ignore-not-found", + "--wait=false", + ]))?; + run_delete(kubectl.command([ + "delete", + "pvc", + "-l", + &selector, + "--ignore-not-found", + "--wait=false", + ]))?; + run_delete(kubectl.command([ + "delete", + "svc", + "-l", + &selector, + "--ignore-not-found", + "--wait=false", + ]))?; + + wait_for_named_resource_deleted( + &kubectl, + "tenant", + &config.tenant_name, + RESOURCE_RESET_TIMEOUT, + )?; + wait_for_selector_empty(&kubectl, "statefulset", &selector, RESOURCE_RESET_TIMEOUT)?; + wait_for_selector_empty(&kubectl, "pod", &selector, RESOURCE_RESET_TIMEOUT)?; + wait_for_selector_empty(&kubectl, "pvc", &selector, RESOURCE_RESET_TIMEOUT)?; + wait_for_selector_empty(&kubectl, "svc", &selector, RESOURCE_RESET_TIMEOUT)?; + + Ok(()) +} + +pub fn cleanup_smoke_tenant_resources(config: &E2eConfig) -> Result<()> { + let kubectl = Kubectl::new(config).namespaced(&config.test_namespace); + let selector = format!("rustfs.tenant={}", config.tenant_name); + + run_best_effort( + kubectl.command([ + "delete", + "tenant", + &config.tenant_name, + "--ignore-not-found", + ]), + "tenant", + ); + run_best_effort( + kubectl.command([ + "delete", + "statefulset", + "-l", + &selector, + "--ignore-not-found", + ]), + "statefulsets", + ); + run_best_effort( + kubectl.command(["delete", "pod", "-l", &selector, "--ignore-not-found"]), + "pods", + ); + run_best_effort( + kubectl.command(["delete", "pvc", "-l", &selector, "--ignore-not-found"]), + "PVCs", + ); + run_best_effort( + kubectl.command(["delete", "svc", "-l", &selector, "--ignore-not-found"]), + "services", + ); + + Ok(()) +} + +fn run_best_effort(command: crate::framework::command::CommandSpec, resource_desc: &str) { + if let Err(error) = command.run() { + println!("best-effort cleanup for {resource_desc} skipped: {error}"); + } +} + +fn namespace_exists(kubectl: &Kubectl, namespace: &str) -> Result { + let output = kubectl.command(["get", "namespace", namespace]).run()?; + Ok(output.code == Some(0)) +} + +fn run_delete(command: CommandSpec) -> Result<()> { + command.run_checked()?; + Ok(()) +} + +fn wait_for_named_resource_deleted( + kubectl: &Kubectl, + resource: &str, + name: &str, + timeout: Duration, +) -> Result<()> { + wait_until(&format!("{resource}/{name} to be deleted"), timeout, || { + let output = kubectl + .command(["get", resource, name, "-o", "name"]) + .run()?; + match output.code { + Some(0) => Ok(false), + _ if is_not_found(&output) => Ok(true), + _ => bail!( + "command failed while waiting for {resource}/{name} deletion\nexit: {:?}\nstdout:\n{}\nstderr:\n{}", + output.code, + output.stdout, + output.stderr + ), + } + }) +} + +fn wait_for_selector_empty( + kubectl: &Kubectl, + resource: &str, + selector: &str, + timeout: Duration, +) -> Result<()> { + wait_until( + &format!("{resource} selector {selector} to be empty"), + timeout, + || { + let output = kubectl + .command([ + "get", + resource, + "-l", + selector, + "-o", + "name", + "--ignore-not-found", + ]) + .run_checked()?; + Ok(output.stdout.lines().all(|line| line.trim().is_empty())) + }, + ) +} + +fn wait_until(description: &str, timeout: Duration, mut condition: F) -> Result<()> +where + F: FnMut() -> Result, +{ + let deadline = Instant::now() + timeout; + loop { + if condition().with_context(|| format!("check {description}"))? { + return Ok(()); + } + + if Instant::now() >= deadline { + bail!("timed out waiting for {description} after {timeout:?}"); + } + + sleep(RESOURCE_RESET_POLL_INTERVAL); + } +} + +fn is_not_found(output: &CommandOutput) -> bool { + output.stderr.contains("NotFound") + || output.stderr.contains("not found") + || output.stdout.contains("NotFound") + || output.stdout.contains("not found") +} + #[cfg(test)] mod tests { use super::{credential_secret_manifest, credential_secret_name, smoke_tenant_manifest}; diff --git a/e2e/src/framework/storage.rs b/e2e/src/framework/storage.rs index 8af1269..a6115b7 100644 --- a/e2e/src/framework/storage.rs +++ b/e2e/src/framework/storage.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::Result; +use anyhow::{Context, Result, ensure}; +use std::thread::sleep; +use std::time::{Duration, Instant}; use crate::framework::{ command::CommandSpec, @@ -21,6 +23,8 @@ use crate::framework::{ }; pub const RUSTFS_RUN_AS_UID: u32 = 10001; +const STORAGE_RESET_TIMEOUT: Duration = Duration::from_secs(120); +const STORAGE_RESET_POLL_INTERVAL: Duration = Duration::from_secs(2); #[derive(Debug, Clone, PartialEq, Eq)] pub struct LocalStorageLayout { @@ -58,6 +62,12 @@ impl LocalStorageLayout { format!("{}-{index}", self.pv_name_prefix) } + fn pv_names(&self) -> Vec { + (1..=self.pv_count) + .map(|index| self.pv_name(index)) + .collect() + } + fn volume_path(&self, index: usize) -> String { format!( "{}/vol{index}", @@ -83,6 +93,22 @@ pub fn volume_directory_commands(config: &E2eConfig) -> Vec { volume_directory_commands_for_layout(config, &LocalStorageLayout::from_config(config)) } +pub fn storage_mount_validation_commands(config: &E2eConfig) -> Vec { + worker_node_names(config) + .into_iter() + .map(|node| { + CommandSpec::new("docker").args([ + "exec".to_string(), + node, + "stat".to_string(), + "-c".to_string(), + "%h".to_string(), + "/mnt/data".to_string(), + ]) + }) + .collect() +} + pub fn volume_directory_commands_for_layout( config: &E2eConfig, layout: &LocalStorageLayout, @@ -176,6 +202,8 @@ pub fn prepare_local_storage_with_layout( config: &E2eConfig, layout: &LocalStorageLayout, ) -> Result<()> { + validate_storage_mounts(config)?; + for command in volume_directory_commands_for_layout(config, layout) { command.run_checked()?; } @@ -186,9 +214,170 @@ pub fn prepare_local_storage_with_layout( Ok(()) } +pub fn reset_default_local_storage(config: &E2eConfig) -> Result<()> { + reset_local_storage_for_layout(config, &LocalStorageLayout::from_config(config)) +} + +pub fn reset_local_storage_for_layout( + config: &E2eConfig, + layout: &LocalStorageLayout, +) -> Result<()> { + validate_storage_mounts(config)?; + validate_reset_layout(layout)?; + delete_local_pvs(config, layout)?; + wait_for_local_pvs_deleted(config, layout, STORAGE_RESET_TIMEOUT)?; + + for command in clean_volume_directory_commands_for_layout(config, layout)? { + command.run_checked()?; + } + + prepare_local_storage_with_layout(config, layout) +} + +fn delete_local_pvs(config: &E2eConfig, layout: &LocalStorageLayout) -> Result<()> { + let pv_names = layout.pv_names(); + if pv_names.is_empty() { + return Ok(()); + } + + println!("deleting dedicated e2e PVs: {}", pv_names.join(", ")); + + let mut args = vec!["delete".to_string(), "pv".to_string()]; + args.extend(pv_names); + args.push("--ignore-not-found".to_string()); + + Kubectl::new(config).command(args).run_checked()?; + Ok(()) +} + +fn wait_for_local_pvs_deleted( + config: &E2eConfig, + layout: &LocalStorageLayout, + timeout: Duration, +) -> Result<()> { + let pv_names = layout.pv_names(); + wait_until( + &format!("PVs {} to be deleted", pv_names.join(", ")), + timeout, + || { + let output = Kubectl::new(config) + .command(["get", "pv", "-o", "name"]) + .run_checked()?; + let existing = output + .stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| line.strip_prefix("persistentvolume/").unwrap_or(line)) + .collect::>(); + + Ok(!pv_names.iter().any(|name| existing.contains(name.as_str()))) + }, + ) +} + +fn clean_volume_directory_commands_for_layout( + config: &E2eConfig, + layout: &LocalStorageLayout, +) -> Result> { + validate_reset_layout(layout)?; + + let mut commands = Vec::new(); + for node in worker_node_names(config) { + for index in 1..=layout.pv_count { + commands.push(clean_volume_directory_command( + &node, + &layout.volume_path(index), + )); + } + } + + Ok(commands) +} + +fn clean_volume_directory_command(node: &str, path: &str) -> CommandSpec { + CommandSpec::new("docker").args([ + "exec".to_string(), + node.to_string(), + "sh".to_string(), + "-c".to_string(), + "mkdir -p \"$1\" && find \"$1\" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && chown -R \"$2:$2\" \"$1\"".to_string(), + "sh".to_string(), + path.to_string(), + RUSTFS_RUN_AS_UID.to_string(), + ]) +} + +fn validate_reset_layout(layout: &LocalStorageLayout) -> Result<()> { + let prefix = layout.volume_path_prefix.trim_end_matches('/'); + ensure!( + prefix == "/mnt/data" || prefix.starts_with("/mnt/data/"), + "refusing to reset e2e storage outside /mnt/data: {}", + layout.volume_path_prefix + ); + ensure!( + !prefix.split('/').any(|part| part == "." || part == ".."), + "refusing to reset unsafe e2e storage path: {}", + layout.volume_path_prefix + ); + + for index in 1..=layout.pv_count { + let path = layout.volume_path(index); + ensure!( + path.starts_with("/mnt/data/"), + "refusing to reset unsafe e2e volume path: {path}" + ); + } + + Ok(()) +} + +fn validate_storage_mounts(config: &E2eConfig) -> Result<()> { + for (node, command) in worker_node_names(config) + .into_iter() + .zip(storage_mount_validation_commands(config)) + { + let output = command.run_checked()?; + let link_count = parse_link_count(&output.stdout)?; + if link_count == 0 { + anyhow::bail!( + "Kind worker {node} has a stale /mnt/data bind mount; the host storage directory was recreated while the cluster was running. Recreate the dedicated e2e cluster with `make e2e-live-create`." + ); + } + } + + Ok(()) +} + +fn parse_link_count(output: &str) -> Result { + Ok(output.trim().parse()?) +} + +fn wait_until(description: &str, timeout: Duration, mut condition: F) -> Result<()> +where + F: FnMut() -> Result, +{ + let deadline = Instant::now() + timeout; + loop { + if condition().with_context(|| format!("check {description}"))? { + return Ok(()); + } + + if Instant::now() >= deadline { + anyhow::bail!("timed out waiting for {description} after {timeout:?}"); + } + + sleep(STORAGE_RESET_POLL_INTERVAL); + } +} + #[cfg(test)] mod tests { - use super::{local_storage_manifest, volume_directory_commands, worker_node_names}; + use super::{ + LocalStorageLayout, clean_volume_directory_commands_for_layout, local_storage_manifest, + parse_link_count, storage_mount_validation_commands, validate_reset_layout, + volume_directory_commands, worker_node_names, + }; use crate::framework::config::E2eConfig; #[test] @@ -226,4 +415,46 @@ mod tests { .contains("docker exec rustfs-e2e-worker mkdir -p /mnt/data/vol1") ); } + + #[test] + fn storage_mount_validation_checks_worker_mount_link_count() { + let config = E2eConfig::defaults(); + + assert!( + storage_mount_validation_commands(&config) + .first() + .expect("at least one command") + .display() + .contains("docker exec rustfs-e2e-worker stat -c %h /mnt/data") + ); + } + + #[test] + fn parse_link_count_trims_stat_output() { + assert_eq!(parse_link_count("2\n").expect("valid link count"), 2); + } + + #[test] + fn reset_layout_rejects_paths_outside_kind_storage_mount() { + let layout = LocalStorageLayout::new("local-storage", "pv", "/var/lib/data", 1); + + assert!(validate_reset_layout(&layout).is_err()); + } + + #[test] + fn clean_volume_commands_clear_only_layout_volume_directories() { + let config = E2eConfig::defaults(); + let layout = LocalStorageLayout::new("local-storage", "pv", "/mnt/data/reset-case", 1); + + let commands = clean_volume_directory_commands_for_layout(&config, &layout) + .expect("commands should render"); + + assert_eq!(commands.len(), 3); + assert!( + commands[0] + .display() + .contains("docker exec rustfs-e2e-worker sh -c") + ); + assert!(commands[0].display().contains("/mnt/data/reset-case/vol1")); + } } diff --git a/e2e/src/framework/tenant_factory.rs b/e2e/src/framework/tenant_factory.rs index 9933046..ca21fc2 100644 --- a/e2e/src/framework/tenant_factory.rs +++ b/e2e/src/framework/tenant_factory.rs @@ -17,12 +17,13 @@ use k8s_openapi::api::core::v1::{ }; use k8s_openapi::apimachinery::pkg::api::resource::Quantity; use operator::types::v1alpha1::k8s::ImagePullPolicy; +use operator::types::v1alpha1::k8s::PodManagementPolicy; use operator::types::v1alpha1::persistence::PersistenceConfig; use operator::types::v1alpha1::pool::{Pool, SchedulingConfig}; use operator::types::v1alpha1::tenant::{Tenant, TenantSpec}; use std::collections::BTreeMap; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct TenantTemplate { pub namespace: String, pub name: String, @@ -31,6 +32,7 @@ pub struct TenantTemplate { pub credential_secret_name: String, pub servers: i32, pub volumes_per_server: i32, + pub pod_management_policy: Option, pub unsafe_bypass_disk_check: bool, } @@ -50,6 +52,7 @@ impl TenantTemplate { credential_secret_name: credential_secret_name.into(), servers: 4, volumes_per_server: 2, + pod_management_policy: Some(PodManagementPolicy::Parallel), unsafe_bypass_disk_check: true, } } @@ -103,6 +106,7 @@ impl TenantTemplate { pools: vec![pool], image: Some(self.image.clone()), image_pull_policy: Some(ImagePullPolicy::IfNotPresent), + pod_management_policy: self.pod_management_policy.clone(), creds_secret: Some(LocalObjectReference { name: self.credential_secret_name.clone(), }), diff --git a/e2e/tests/cert_manager_tls.rs b/e2e/tests/cert_manager_tls.rs index f0597d1..00ba9f5 100644 --- a/e2e/tests/cert_manager_tls.rs +++ b/e2e/tests/cert_manager_tls.rs @@ -876,7 +876,7 @@ fn cert_manager_artifacts_do_not_expose_secret_material() -> Result<()> { } #[tokio::test] -#[ignore = "requires cert-manager installed in the dedicated Kind cluster; run after `make e2e-live-run`"] +#[ignore = "requires cert-manager installed in the dedicated Kind cluster; run through `make e2e-live-run`"] async fn cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring() -> Result<()> { let base_config = E2eConfig::from_env(); live::require_live_enabled(&base_config)?; @@ -915,7 +915,7 @@ async fn cert_manager_managed_certificate_reaches_tls_ready_and_https_wiring() - } #[tokio::test] -#[ignore = "creates an external TLS Secret and waits for rollout; run after `make e2e-live-run`"] +#[ignore = "creates an external TLS Secret and waits for rollout; run through `make e2e-live-run`"] async fn cert_manager_external_secret_reaches_tls_ready_and_rolls_on_secret_hash() -> Result<()> { let base_config = E2eConfig::from_env(); live::require_live_enabled(&base_config)?; @@ -957,7 +957,7 @@ async fn cert_manager_external_secret_reaches_tls_ready_and_rolls_on_secret_hash } #[tokio::test] -#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +#[ignore = "mutates live Tenant fixtures; run through `make e2e-live-run`"] async fn cert_manager_rejects_secret_missing_tls_crt() -> Result<()> { assert_negative_case_tls_reason( tls_e2e::NegativeTlsCase::MissingTlsCrt, @@ -967,7 +967,7 @@ async fn cert_manager_rejects_secret_missing_tls_crt() -> Result<()> { } #[tokio::test] -#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +#[ignore = "mutates live Tenant fixtures; run through `make e2e-live-run`"] async fn cert_manager_rejects_secret_missing_tls_key() -> Result<()> { assert_negative_case_tls_reason( tls_e2e::NegativeTlsCase::MissingTlsKey, @@ -977,7 +977,7 @@ async fn cert_manager_rejects_secret_missing_tls_key() -> Result<()> { } #[tokio::test] -#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +#[ignore = "mutates live Tenant fixtures; run through `make e2e-live-run`"] async fn cert_manager_rejects_secret_missing_ca_for_internode_https() -> Result<()> { assert_negative_case_tls_reason( tls_e2e::NegativeTlsCase::MissingCaForInternodeHttps, @@ -987,7 +987,7 @@ async fn cert_manager_rejects_secret_missing_ca_for_internode_https() -> Result< } #[tokio::test] -#[ignore = "requires cert-manager API and mutates live Tenant fixtures; run after `make e2e-live-run`"] +#[ignore = "requires cert-manager API and mutates live Tenant fixtures; run through `make e2e-live-run`"] async fn cert_manager_rejects_missing_issuer_for_managed_certificate() -> Result<()> { assert_negative_case_tls_reason( tls_e2e::NegativeTlsCase::MissingIssuer, @@ -997,7 +997,7 @@ async fn cert_manager_rejects_missing_issuer_for_managed_certificate() -> Result } #[tokio::test] -#[ignore = "requires cert-manager API and mutates live Tenant fixtures; run after `make e2e-live-run`"] +#[ignore = "requires cert-manager API and mutates live Tenant fixtures; run through `make e2e-live-run`"] async fn cert_manager_reports_pending_certificate_not_ready() -> Result<()> { assert_negative_case_tls_reason( tls_e2e::NegativeTlsCase::PendingCertificate, @@ -1007,7 +1007,7 @@ async fn cert_manager_reports_pending_certificate_not_ready() -> Result<()> { } #[tokio::test] -#[ignore = "mutates live Tenant fixtures; run after `make e2e-live-run`"] +#[ignore = "mutates live Tenant fixtures; run through `make e2e-live-run`"] async fn cert_manager_rejects_hot_reload() -> Result<()> { assert_negative_case_tls_reason( tls_e2e::NegativeTlsCase::HotReloadUnsupported, diff --git a/e2e/tests/faults.rs b/e2e/tests/faults.rs index c3b3252..b531cfc 100644 --- a/e2e/tests/faults.rs +++ b/e2e/tests/faults.rs @@ -24,7 +24,7 @@ fn faults_are_not_destructive_without_explicit_opt_in() { } #[test] -#[ignore = "reserved for destructive fault scenarios; no public live target in the reduced workflow"] +#[ignore = "reserved for destructive fault scenarios; run through `make e2e-live-faults`"] fn fault_live_suite_requires_explicit_destructive_opt_in() -> Result<()> { let config = E2eConfig::from_env(); diff --git a/e2e/tests/sts_functional.rs b/e2e/tests/sts_functional.rs new file mode 100644 index 0000000..3855e96 --- /dev/null +++ b/e2e/tests/sts_functional.rs @@ -0,0 +1,547 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use anyhow::{Context, Result, bail, ensure}; +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use k8s_openapi::api::core::v1 as corev1; +use kube::Api; +use operator::{ + console::state::AppState, + sts::{rustfs_client::RustfsAdminClient, server::routes}, + types::v1alpha1::tenant::Tenant, +}; +use rustfs_operator_e2e::framework::{ + artifacts::ArtifactCollector, + cert_manager_tls as tls_e2e, + config::E2eConfig, + kube_client, + kubectl::Kubectl, + live, + port_forward::{PortForwardGuard, PortForwardSpec}, + resources, wait, +}; +use std::{ + net::{Ipv4Addr, SocketAddr}, + time::{Duration, Instant}, +}; +use tokio::time::sleep; +use tower::ServiceExt; + +const VALID_WEB_IDENTITY_FORM: &str = + "Version=2011-06-15&Action=AssumeRoleWithWebIdentity&WebIdentityToken=service-account-token"; +const STS_LIVE_AUDIENCE: &str = "sts.rustfs.com"; +const STS_LIVE_SERVICE_ACCOUNT: &str = "sts-e2e-workload"; +const STS_LIVE_POLICY_BINDING: &str = "sts-e2e-binding"; +const STS_LIVE_POLICY: &str = "sts-e2e-readonly"; +const OPERATOR_STS_TLS_SECRET: &str = "sts-tls"; +const PORT_FORWARD_READY_TIMEOUT: Duration = Duration::from_secs(120); +const STS_LIVE_POLICY_DOCUMENT: &str = r#"{"Version":"2012-10-17","Statement":[{"Sid":"RustfsStsE2eReadOnly","Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::*","arn:aws:s3:::*/*"]}]}"#; + +#[tokio::test] +async fn sts_requires_explicit_namespace_and_tenant_route() { + let app = routes().with_state(AppState::new("test-secret".to_string())); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/sts/rustfs-e2e") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from(VALID_WEB_IDENTITY_FORM)) + .expect("request builds"), + ) + .await + .expect("handler responds"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn sts_explicit_tenant_route_accepts_valid_form_shape() { + let app = routes().with_state(AppState::new("test-secret".to_string())); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/sts/rustfs-e2e/e2e-tenant") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from(VALID_WEB_IDENTITY_FORM)) + .expect("request builds"), + ) + .await + .expect("handler responds"); + + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); + let body = response_text(response).await; + assert!(body.contains("NotImplemented")); +} + +#[tokio::test] +async fn sts_explicit_tenant_route_returns_sts_xml_validation_errors() { + let app = routes().with_state(AppState::new("test-secret".to_string())); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/sts/rustfs-e2e/e2e-tenant") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from( + "Version=2011-06-15&Action=AssumeRoleWithWebIdentity", + )) + .expect("request builds"), + ) + .await + .expect("handler responds"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response_text(response).await; + assert!(body.contains("MissingParameter")); + assert!(body.contains("WebIdentityToken")); +} + +#[tokio::test] +#[ignore = "requires live Kubernetes TokenReview, PolicyBinding CRD, and RustFS STS; run through `make e2e-live-run`"] +async fn sts_live_tokenreview_policybinding_and_assume_role_succeeds() -> Result<()> { + let config = E2eConfig::from_env(); + live::require_live_enabled(&config)?; + live::ensure_dedicated_context(&config)?; + + let result = run_sts_live_flow(&config).await; + + if let Err(error) = &result { + let collector = ArtifactCollector::new(&config.artifacts_dir); + match collector.collect_kubernetes_snapshot( + "sts_live_tokenreview_policybinding_and_assume_role_succeeds", + &config, + ) { + Ok(report) => { + eprintln!("collected e2e artifacts under {}", report.dir.display()); + eprintln!("{}", report.diagnosis); + } + Err(artifact_error) => { + eprintln!("failed to collect e2e artifacts after {error}: {artifact_error}"); + } + } + } + + result +} + +#[tokio::test] +#[ignore = "requires live Kubernetes TokenReview, PolicyBinding CRD, and operator STS TLS; run through `make e2e-live-run`"] +async fn sts_live_rejects_non_tls_tenant() -> Result<()> { + let config = E2eConfig::from_env(); + live::require_live_enabled(&config)?; + live::ensure_dedicated_context(&config)?; + + let result = run_sts_non_tls_rejection_flow(&config).await; + + if let Err(error) = &result { + let collector = ArtifactCollector::new(&config.artifacts_dir); + match collector.collect_kubernetes_snapshot("sts_live_rejects_non_tls_tenant", &config) { + Ok(report) => { + eprintln!("collected e2e artifacts under {}", report.dir.display()); + eprintln!("{}", report.diagnosis); + } + Err(artifact_error) => { + eprintln!("failed to collect e2e artifacts after {error}: {artifact_error}"); + } + } + } + + result +} + +async fn response_text(response: axum::response::Response) -> String { + let body = to_bytes(response.into_body(), usize::MAX) + .await + .expect("body reads"); + String::from_utf8(body.to_vec()).expect("response body is utf8") +} + +async fn run_sts_live_flow(config: &E2eConfig) -> Result<()> { + let config = tls_e2e::managed_certificate_case_config(config); + let kubectl = Kubectl::new(&config); + tls_e2e::apply_managed_certificate_case_resources(&config)?; + + let kube_client = kube_client::default_client().await?; + tls_e2e::wait_for_tenant_tls_ready( + kube_client.clone(), + &config.test_namespace, + &config.tenant_name, + tls_e2e::positive_cert_manager_tls_timeout(&config), + ) + .await + .context("wait for TLS-enabled Tenant for STS live e2e")?; + tls_e2e::wait_for_certificate_ready( + kube_client.clone(), + &config.test_namespace, + &tls_e2e::managed_certificate_name(&config), + tls_e2e::positive_cert_manager_tls_timeout(&config), + ) + .await + .context("wait for operator-managed Tenant certificate")?; + + let tenants = kube_client::tenant_api(kube_client.clone(), &config.test_namespace); + let tenant = wait::wait_for_tenant_ready(tenants, &config.tenant_name, config.timeout) + .await + .context("reuse Ready TLS Tenant for STS live e2e")?; + + kubectl + .apply_yaml_command(sts_live_service_account_manifest(&config.test_namespace)) + .run_checked() + .context("apply STS live ServiceAccount")?; + kubectl + .apply_yaml_command(sts_live_policy_binding_manifest(&config.test_namespace)) + .run_checked() + .context("apply STS live PolicyBinding")?; + + ensure_rustfs_canned_policy(&config, &tenant, &kube_client).await?; + let token = create_sts_live_service_account_token(&config)?; + + let (sts_url, sts_client, _sts_port_forward) = + start_operator_sts_https_port_forward(&config, &kube_client).await?; + + let response = assume_role_with_web_identity(&sts_url, &config, &token, &sts_client).await?; + assert_sts_live_response(&config, &response)?; + + Ok(()) +} + +async fn run_sts_non_tls_rejection_flow(config: &E2eConfig) -> Result<()> { + resources::apply_smoke_tenant_resources(config).context("apply non-TLS smoke Tenant")?; + let kubectl = Kubectl::new(config); + let kube_client = kube_client::default_client().await?; + let tenants = kube_client::tenant_api(kube_client.clone(), &config.test_namespace); + wait::wait_for_tenant_ready(tenants, &config.tenant_name, config.timeout) + .await + .context("wait for non-TLS smoke Tenant")?; + + kubectl + .apply_yaml_command(sts_live_service_account_manifest(&config.test_namespace)) + .run_checked() + .context("apply STS live ServiceAccount")?; + kubectl + .apply_yaml_command(sts_live_policy_binding_manifest(&config.test_namespace)) + .run_checked() + .context("apply STS live PolicyBinding")?; + let token = create_sts_live_service_account_token(config)?; + + let (sts_url, sts_client, _sts_port_forward) = + start_operator_sts_https_port_forward(config, &kube_client).await?; + let (status, body) = + assume_role_with_web_identity_response(&sts_url, config, &token, &sts_client).await?; + + ensure!( + status == StatusCode::BAD_REQUEST, + "non-TLS Tenant STS request should fail with BAD_REQUEST, got {status}:\n{body}" + ); + ensure!( + body.contains("InvalidParameterValue") && body.contains("tenantTls"), + "non-TLS Tenant STS request should identify tenantTls as invalid:\n{body}" + ); + + Ok(()) +} + +async fn ensure_rustfs_canned_policy( + config: &E2eConfig, + tenant: &Tenant, + kube_client: &kube::Client, +) -> Result<()> { + let rustfs_port_forward_spec = + PortForwardSpec::tenant_io(&config.test_namespace, &config.tenant_name); + let rustfs_host = rustfs_service_dns(&config.test_namespace, &config.tenant_name); + let rustfs_url = local_https_base_url(&rustfs_host, &rustfs_port_forward_spec); + let mut rustfs_port_forward = + PortForwardSpec::start_tenant_io(config).context("start RustFS tenant IO port-forward")?; + let tenant_ca = RustfsAdminClient::load_tenant_tls_ca(kube_client, tenant) + .await + .context("load TLS Tenant CA")? + .context("TLS Tenant should publish a CA Secret reference")?; + let rustfs_probe_client = tls_client( + &tenant_ca, + &rustfs_host, + rustfs_port_forward_spec.local_port, + )?; + wait_for_port_forward(&mut rustfs_port_forward, &rustfs_url, &rustfs_probe_client).await?; + + let credentials = RustfsAdminClient::load_tenant_credentials(kube_client, tenant) + .await + .context("load RustFS tenant credentials")?; + let rustfs_admin = RustfsAdminClient::new_with_base_url_and_http_client( + rustfs_url, + credentials.access_key, + credentials.secret_key, + rustfs_probe_client, + ); + + rustfs_admin + .add_canned_policy(STS_LIVE_POLICY, STS_LIVE_POLICY_DOCUMENT) + .await + .context("add RustFS canned policy for STS live e2e")?; + + Ok(()) +} + +fn create_sts_live_service_account_token(config: &E2eConfig) -> Result { + let output = Kubectl::new(config) + .namespaced(&config.test_namespace) + .command(vec![ + "create".to_string(), + "token".to_string(), + STS_LIVE_SERVICE_ACCOUNT.to_string(), + format!("--audience={STS_LIVE_AUDIENCE}"), + "--duration=10m".to_string(), + ]) + .run_checked() + .context("create projected ServiceAccount token for STS live e2e")?; + let token = output.stdout.trim().to_string(); + + ensure!( + !token.is_empty(), + "kubectl create token returned an empty token" + ); + Ok(token) +} + +async fn assume_role_with_web_identity( + sts_url: &str, + config: &E2eConfig, + token: &str, + client: &reqwest::Client, +) -> Result { + let (status, body) = + assume_role_with_web_identity_response(sts_url, config, token, client).await?; + + ensure!( + status.is_success(), + "operator STS returned {status}:\n{body}" + ); + + Ok(body) +} + +async fn assume_role_with_web_identity_response( + sts_url: &str, + config: &E2eConfig, + token: &str, + client: &reqwest::Client, +) -> Result<(StatusCode, String)> { + let response = client + .post(format!( + "{}/sts/{}/{}", + sts_url.trim_end_matches('/'), + config.test_namespace, + config.tenant_name + )) + .form(&[ + ("Version", "2011-06-15"), + ("Action", "AssumeRoleWithWebIdentity"), + ("WebIdentityToken", token), + ("DurationSeconds", "900"), + ]) + .send() + .await + .context("send AssumeRoleWithWebIdentity request to operator STS")?; + let status = response.status(); + let body = response + .text() + .await + .context("read operator STS response body")?; + + Ok((status, body)) +} + +fn assert_sts_live_response(config: &E2eConfig, body: &str) -> Result<()> { + let subject = format!( + "system:serviceaccount:{}:{}", + config.test_namespace, STS_LIVE_SERVICE_ACCOUNT + ); + let audience = format!("{STS_LIVE_AUDIENCE}"); + let role_arn = format!( + "arn:rustfs:sts::{}:assumed-role/{}/{}:{}", + config.test_namespace, config.tenant_name, config.test_namespace, STS_LIVE_SERVICE_ACCOUNT + ); + + ensure!( + body.contains(""), + "missing AccessKeyId in operator STS success response" + ); + ensure!( + body.contains(""), + "missing SecretAccessKey in operator STS success response" + ); + ensure!( + body.contains(""), + "missing SessionToken in operator STS success response" + ); + ensure!( + body.contains("kubernetes"), + "missing provider in operator STS success response" + ); + + Ok(()) +} + +async fn start_operator_sts_https_port_forward( + config: &E2eConfig, + kube_client: &kube::Client, +) -> Result<(String, reqwest::Client, PortForwardGuard)> { + let sts_port_forward_spec = PortForwardSpec::operator_sts(&config.operator_namespace); + let sts_host = operator_sts_service_dns(&config.operator_namespace); + let sts_url = local_https_base_url(&sts_host, &sts_port_forward_spec); + let mut sts_port_forward = + PortForwardSpec::start_operator_sts(config).context("start operator STS port-forward")?; + let sts_ca = load_secret_key( + kube_client, + &config.operator_namespace, + OPERATOR_STS_TLS_SECRET, + "ca.crt", + ) + .await + .context("load operator STS CA")?; + let sts_client = tls_client(&sts_ca, &sts_host, sts_port_forward_spec.local_port)?; + wait_for_port_forward(&mut sts_port_forward, &sts_url, &sts_client).await?; + + Ok((sts_url, sts_client, sts_port_forward)) +} + +async fn wait_for_port_forward( + port_forward: &mut PortForwardGuard, + base_url: &str, + client: &reqwest::Client, +) -> Result<()> { + let deadline = Instant::now() + PORT_FORWARD_READY_TIMEOUT; + + loop { + port_forward.ensure_running()?; + let last_error = match client.get(base_url).send().await { + Ok(_) => return Ok(()), + Err(error) => error.to_string(), + }; + + if Instant::now() >= deadline { + bail!( + "port-forward not ready after {:?}; last error: {}; command: {}; log {}:\n{}", + PORT_FORWARD_READY_TIMEOUT, + last_error, + port_forward.command_display(), + port_forward.log_path().display(), + port_forward.log_contents() + ); + } + + sleep(Duration::from_secs(1)).await; + } +} + +fn local_https_base_url(host: &str, spec: &PortForwardSpec) -> String { + format!("https://{host}:{}", spec.local_port) +} + +fn rustfs_service_dns(namespace: &str, tenant_name: &str) -> String { + format!("{tenant_name}-io.{namespace}.svc") +} + +fn operator_sts_service_dns(namespace: &str) -> String { + format!("rustfs-operator-sts.{namespace}.svc") +} + +fn tls_client(ca_pem: &[u8], host: &str, local_port: u16) -> Result { + operator::install_rustls_crypto_provider(); + + let certs = reqwest::Certificate::from_pem_bundle(ca_pem) + .context("parse CA PEM bundle for TLS client")?; + let local_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, local_port)); + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(15)) + .resolve(host, local_addr); + for cert in certs { + builder = builder.add_root_certificate(cert); + } + Ok(builder.build()?) +} + +async fn load_secret_key( + kube_client: &kube::Client, + namespace: &str, + secret_name: &str, + key: &str, +) -> Result> { + let api: Api = Api::namespaced(kube_client.clone(), namespace); + let secret = api + .get(secret_name) + .await + .with_context(|| format!("load Secret {namespace}/{secret_name}"))?; + + secret + .data + .as_ref() + .and_then(|data| data.get(key)) + .map(|bytes| bytes.0.clone()) + .filter(|bytes| !bytes.is_empty()) + .with_context(|| format!("Secret {namespace}/{secret_name} missing non-empty key {key}")) +} + +fn sts_live_service_account_manifest(namespace: &str) -> String { + format!( + r#"apiVersion: v1 +kind: ServiceAccount +metadata: + name: {STS_LIVE_SERVICE_ACCOUNT} + namespace: {namespace} +"# + ) +} + +fn sts_live_policy_binding_manifest(namespace: &str) -> String { + format!( + r#"apiVersion: sts.rustfs.com/v1alpha1 +kind: PolicyBinding +metadata: + name: {STS_LIVE_POLICY_BINDING} + namespace: {namespace} +spec: + application: + namespace: {namespace} + serviceaccount: {STS_LIVE_SERVICE_ACCOUNT} + policies: + - {STS_LIVE_POLICY} +"# + ) +} diff --git a/e2e/tests/sts_manifest.rs b/e2e/tests/sts_manifest.rs new file mode 100644 index 0000000..57c2752 --- /dev/null +++ b/e2e/tests/sts_manifest.rs @@ -0,0 +1,216 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde_yaml_ng::Value; +use std::process::{Command, Output}; + +#[test] +fn k8s_dev_manifests_expose_sts_service_and_rbac_permissions() { + // CRD/STS-specific RBAC and porting is required for STS flow. + let k8s_rbac = std::fs::read_to_string("../deploy/k8s-dev/operator-rbac.yaml") + .expect("k8s dev operator-rbac exists"); + let k8s_deploy = std::fs::read_to_string("../deploy/k8s-dev/operator-deployment.yaml") + .expect("k8s dev operator deployment exists"); + let k8s_sts_svc = std::fs::read_to_string("../deploy/k8s-dev/operator-sts-service.yaml") + .expect("k8s dev sts service exists"); + + assert!( + k8s_rbac.contains("policybindings"), + "k8s-rbac should include policybindings" + ); + assert!( + k8s_rbac.contains("tokenreviews"), + "k8s-rbac should include tokenreviews" + ); + assert!(k8s_deploy.contains("app.kubernetes.io/component: operator")); + assert!(k8s_deploy.contains("name: sts")); + assert!(k8s_deploy.contains("containerPort: 4223")); + assert!(k8s_deploy.contains("name: OPERATOR_STS_ENABLED")); + assert!(k8s_deploy.contains("value: \"true\"")); + assert!(k8s_deploy.contains("name: OPERATOR_STS_AUDIENCE")); + assert!(k8s_deploy.contains("value: sts.rustfs.com")); + assert!(k8s_deploy.contains("value: \"4223\"")); + assert!(k8s_deploy.contains("name: OPERATOR_NAMESPACE")); + assert!(k8s_deploy.contains("fieldPath: metadata.namespace")); + assert!(k8s_deploy.contains("name: OPERATOR_STS_SERVICE_NAME")); + assert!(k8s_deploy.contains("value: rustfs-operator-sts")); + assert!(k8s_deploy.contains("name: OPERATOR_STS_TLS_ENABLED")); + assert!(k8s_deploy.contains("name: OPERATOR_STS_TLS_AUTO")); + assert!(!k8s_deploy.contains("name: OPERATOR_STS_TLS_SECRET_NAME")); + assert!(k8s_sts_svc.contains("name: rustfs-operator-sts")); + assert!(k8s_sts_svc.contains("targetPort: sts")); + + // Ensure k8s dev manifests stay valid YAML after additions. + assert_yaml_documents_parse(&k8s_rbac, "operator-rbac"); + assert_yaml_documents_parse(&k8s_deploy, "operator-deployment"); + assert_yaml_documents_parse(&k8s_sts_svc, "operator-sts-service"); +} + +#[test] +fn helm_sts_template_and_values_are_consistent() { + let helm_values = std::fs::read_to_string("../deploy/rustfs-operator/values.yaml") + .expect("helm values exists"); + let helm_deploy = + std::fs::read_to_string("../deploy/rustfs-operator/templates/deployment.yaml") + .expect("helm deployment template exists"); + let helm_sts_svc = + std::fs::read_to_string("../deploy/rustfs-operator/templates/operator-sts-service.yaml") + .expect("helm sts service template exists"); + let helm_clusterrole = + std::fs::read_to_string("../deploy/rustfs-operator/templates/clusterrole.yaml") + .expect("helm clusterrole template exists"); + + let sts_values = helm_values + .split("# ServiceAccount configuration") + .next() + .expect("values contain sts section before service account"); + assert!(sts_values.contains("sts:")); + assert!(sts_values.contains("enabled: true")); + assert!(sts_values.contains("audience: sts.rustfs.com")); + assert!(sts_values.contains("port: 4223")); + assert!(sts_values.contains("tls:")); + assert!(!sts_values.contains("secretName:")); + assert!(!sts_values.contains("nodePort:")); + assert!(!sts_values.contains("loadBalancerIP:")); + assert!(!helm_values.contains("OPERATOR_STS_PORT")); + + assert!(helm_deploy.contains("app.kubernetes.io/component: operator")); + assert!(helm_deploy.contains("{{- if .Values.sts.enabled }}")); + assert!(helm_deploy.contains("name: sts")); + assert!(helm_deploy.contains("containerPort: {{ .Values.sts.port }}")); + assert!(helm_deploy.contains("name: OPERATOR_STS_ENABLED")); + assert!(helm_deploy.contains("value: {{ .Values.sts.enabled | quote }}")); + assert!(helm_deploy.contains("name: OPERATOR_STS_AUDIENCE")); + assert!(helm_deploy.contains("value: {{ .Values.sts.audience | quote }}")); + assert!(helm_deploy.contains("name: OPERATOR_STS_PORT")); + assert!(helm_deploy.contains("value: {{ .Values.sts.port | quote }}")); + assert!(helm_deploy.contains("name: OPERATOR_NAMESPACE")); + assert!(helm_deploy.contains("fieldPath: metadata.namespace")); + assert!(helm_deploy.contains("name: OPERATOR_STS_SERVICE_NAME")); + assert!( + helm_deploy + .contains("{{ printf \"%s-sts\" (include \"rustfs-operator.fullname\" .) | quote }}") + ); + assert!(helm_deploy.contains("name: OPERATOR_STS_TLS_ENABLED")); + assert!(helm_deploy.contains("value: {{ .Values.sts.tls.enabled | quote }}")); + assert!(helm_deploy.contains("name: OPERATOR_STS_TLS_AUTO")); + assert!(helm_deploy.contains("value: {{ .Values.sts.tls.auto | quote }}")); + assert!(!helm_deploy.contains("name: OPERATOR_STS_TLS_SECRET_NAME")); + + assert!(helm_clusterrole.contains("policybindings")); + assert!(helm_clusterrole.contains("tokenreviews")); + + assert!(helm_sts_svc.contains("{{ include \"rustfs-operator.fullname\" . }}-sts")); + assert!(helm_sts_svc.contains("targetPort: sts")); + assert!(helm_sts_svc.contains("app.kubernetes.io/component: operator")); + assert!(helm_sts_svc.contains("operator STS currently supports only ClusterIP")); + assert!(!helm_sts_svc.contains("nodePort:")); + assert!(!helm_sts_svc.contains("loadBalancerIP:")); + + // Static assertions keep the value/template contract visible even when helm is unavailable. + assert!(helm_sts_svc.contains("{{- if .Values.sts.enabled -}}")); +} + +#[test] +fn helm_template_renders_sts_enabled_disabled_and_rejects_external_plaintext() { + let Some(default_render) = helm_template(&[]) else { + return; + }; + + assert!( + default_render.status.success(), + "default helm template should render successfully: {}", + String::from_utf8_lossy(&default_render.stderr) + ); + let default_stdout = String::from_utf8(default_render.stdout).expect("helm stdout is utf8"); + assert!(default_stdout.contains("name: rustfs-operator-sts")); + assert!(default_stdout.contains("name: OPERATOR_STS_ENABLED")); + assert!(default_stdout.contains("value: \"true\"")); + assert!(default_stdout.contains("name: OPERATOR_STS_AUDIENCE")); + assert!(default_stdout.contains("value: \"sts.rustfs.com\"")); + assert!(default_stdout.contains("name: OPERATOR_STS_PORT")); + assert!(default_stdout.contains("name: OPERATOR_STS_TLS_ENABLED")); + assert!(default_stdout.contains("value: \"true\"")); + assert!(default_stdout.contains("name: OPERATOR_STS_TLS_AUTO")); + assert!(!default_stdout.contains("name: OPERATOR_STS_TLS_SECRET_NAME")); + assert_yaml_documents_parse(&default_stdout, "helm-default-render"); + + let Some(disabled_render) = helm_template(&["--set", "sts.enabled=false"]) else { + return; + }; + assert!( + disabled_render.status.success(), + "disabled helm template should render successfully: {}", + String::from_utf8_lossy(&disabled_render.stderr) + ); + let disabled_stdout = + String::from_utf8(disabled_render.stdout).expect("disabled helm stdout is utf8"); + assert!(!disabled_stdout.contains("name: rustfs-operator-sts")); + assert!(disabled_stdout.contains("name: OPERATOR_STS_ENABLED")); + assert!(disabled_stdout.contains("value: \"false\"")); + assert!(!disabled_stdout.contains("name: OPERATOR_STS_PORT")); + assert!(!disabled_stdout.contains("name: OPERATOR_STS_TLS_ENABLED")); + assert_yaml_documents_parse(&disabled_stdout, "helm-disabled-render"); + + let Some(external_render) = helm_template(&["--set", "sts.service.type=NodePort"]) else { + return; + }; + assert!( + !external_render.status.success(), + "NodePort STS should fail until TLS termination is configured" + ); + let external_stderr = String::from_utf8_lossy(&external_render.stderr); + assert!(external_stderr.contains("operator STS currently supports only ClusterIP")); +} + +fn helm_template(args: &[&str]) -> Option { + if !helm_is_available() { + eprintln!("skipping helm template assertions: helm binary is not available"); + return None; + } + + let mut command = Command::new("helm"); + command.args(["template", "rustfs-operator", "../deploy/rustfs-operator"]); + command.args(args); + + Some(command.output().expect("helm template command runs")) +} + +fn helm_is_available() -> bool { + Command::new("helm") + .args(["version", "--short"]) + .output() + .map(|output| output.status.success()) + .unwrap_or(false) +} + +fn assert_yaml_documents_parse(yaml: &str, name: &str) { + let mut had_content = false; + + for raw_doc in yaml.split("---") { + if raw_doc.trim().is_empty() { + continue; + } + + serde_yaml_ng::from_str::(raw_doc).unwrap_or_else(|error| { + panic!("{name} contains invalid yaml document: {error}"); + }); + had_content = true; + } + + assert!( + had_content, + "{name} should contain at least one yaml document" + ); +} diff --git a/src/console/server.rs b/src/console/server.rs index 806c016..5c24330 100755 --- a/src/console/server.rs +++ b/src/console/server.rs @@ -50,13 +50,27 @@ fn cors_allowed_origins() -> Vec { /// Start the Console HTTP server (Axum). pub async fn run(port: u16) -> Result<(), Box> { + crate::install_rustls_crypto_provider(); + tracing::info!("Starting RustFS Operator Console on port {}", port); // JWT signing secret (set JWT_SECRET in production) let jwt_secret = std::env::var("JWT_SECRET") .unwrap_or_else(|_| "rustfs-console-secret-change-me-in-production".to_string()); - let state = AppState::new(jwt_secret); + let state = match Client::try_default().await { + Ok(kube_client) => { + tracing::info!("Kubernetes client initialized for STS authorization flow"); + AppState::new(jwt_secret).with_kube_client(kube_client) + } + Err(error) => { + tracing::warn!( + "Kubernetes client unavailable; STS authorization paths fall back to compatibility mode: {}", + error + ); + AppState::new(jwt_secret) + } + }; let cors_origins = cors_allowed_origins(); diff --git a/src/console/state.rs b/src/console/state.rs index 39f2cac..06c02fd 100755 --- a/src/console/state.rs +++ b/src/console/state.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use kube::Client; use std::sync::Arc; /// Shared Axum application state. @@ -21,6 +22,11 @@ use std::sync::Arc; pub struct AppState { /// Symmetric key for signing session JWTs pub jwt_secret: Arc, + + /// Optional Kubernetes client used by control-plane APIs that need cluster access. + /// + /// Most unit tests run without a live cluster, so this is optional. + pub kube_client: Option, } impl AppState { @@ -28,8 +34,15 @@ impl AppState { pub fn new(jwt_secret: String) -> Self { Self { jwt_secret: Arc::new(jwt_secret), + kube_client: None, } } + + /// Attach a Kubernetes client for request handlers that need cluster reads. + pub fn with_kube_client(mut self, kube_client: Client) -> Self { + self.kube_client = Some(kube_client); + self + } } /// JWT Claims diff --git a/src/lib.rs b/src/lib.rs index 650b719..aebb0f6 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,8 +16,14 @@ use crate::context::Context; use crate::reconcile::{error_policy, reconcile_rustfs}; +use crate::types::v1alpha1::policy_binding::PolicyBinding; use crate::types::v1alpha1::tenant::Tenant; +use axum::{Router, body::Body}; use futures::StreamExt; +use hyper::body::Incoming; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto::Builder as HyperBuilder; +use hyper_util::service::TowerToHyperService; use k8s_openapi::api::apps::v1 as appsv1; use k8s_openapi::api::core::v1 as corev1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; @@ -29,6 +35,8 @@ use std::collections::BTreeMap; use std::pin::Pin; use std::sync::Arc; use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tokio_rustls::TlsAcceptor; +use tower::ServiceExt as _; use tracing::{info, warn}; const RUSTFS_TENANT_LABEL: &str = "rustfs.tenant"; @@ -37,6 +45,10 @@ const CERT_MANAGER_VERSION: &str = "v1"; const CERT_MANAGER_CERTIFICATE_KIND: &str = "Certificate"; const CERT_MANAGER_CERTIFICATE_PLURAL: &str = "certificates"; +pub fn install_rustls_crypto_provider() { + let _ = rustls::crypto::ring::default_provider().install_default(); +} + mod context; pub mod reconcile; mod status; @@ -45,11 +57,14 @@ pub mod utils; // Console module (Web UI) pub mod console; +pub mod sts; #[cfg(test)] pub mod tests; pub async fn run() -> Result<(), Box> { + install_rustls_crypto_provider(); + tracing_subscriber::fmt() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .with_level(true) @@ -59,8 +74,32 @@ pub async fn run() -> Result<(), Box> { .init(); let client = Client::try_default().await?; - let tenant_client = Api::::all(client.clone()); + if operator_sts_enabled() { + let sts_port = operator_sts_port(); + let sts_state = + crate::console::state::AppState::new(String::new()).with_kube_client(client.clone()); + let sts_tls_config = crate::sts::tls::OperatorStsTlsConfig::from_env(); + let tls_server_config = if sts_tls_config.enabled { + let material = + crate::sts::tls::load_or_create_sts_tls_material(&client, &sts_tls_config).await?; + Some(Arc::new(crate::sts::tls::build_tls_server_config( + &material, + )?)) + } else { + warn!("Operator STS TLS disabled by OPERATOR_STS_TLS_ENABLED=false"); + None + }; + let sts_listener = bind_sts_listener(sts_port, tls_server_config.is_some()).await?; + tokio::spawn(async move { + if let Err(error) = run_sts_server(sts_listener, sts_state, tls_server_config).await { + warn!(%error, "Operator STS server stopped unexpectedly"); + } + }); + } else { + tracing::info!("Operator STS server disabled by OPERATOR_STS_ENABLED=false"); + } + let tenant_client = Api::::all(client.clone()); let context = Context::new(client.clone()); let controller = Controller::new(tenant_client, watcher::Config::default()) .owns( @@ -118,6 +157,113 @@ pub async fn run() -> Result<(), Box> { Ok(()) } +fn operator_sts_port() -> u16 { + let default_port: u16 = 4223; + match std::env::var("OPERATOR_STS_PORT") { + Ok(raw_port) => match raw_port.parse::() { + Ok(port) => port, + Err(error) => { + warn!( + %error, + raw_port, + "invalid OPERATOR_STS_PORT value, using default" + ); + default_port + } + }, + Err(_) => default_port, + } +} + +fn operator_sts_enabled() -> bool { + match std::env::var("OPERATOR_STS_ENABLED") { + Ok(value) => match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => true, + "0" | "false" | "no" | "off" => false, + _ => { + warn!( + value, + "invalid OPERATOR_STS_ENABLED value, defaulting to enabled" + ); + true + } + }, + Err(_) => true, + } +} + +async fn bind_sts_listener( + port: u16, + tls_enabled: bool, +) -> Result> { + let addr = std::net::SocketAddr::from(([0, 0, 0, 0], port)); + let listener = tokio::net::TcpListener::bind(addr).await?; + let scheme = if tls_enabled { "https" } else { "http" }; + tracing::info!("Operator STS server listening on {}://{}", scheme, addr); + Ok(listener) +} + +async fn run_sts_server( + listener: tokio::net::TcpListener, + state: crate::console::state::AppState, + tls_config: Option>, +) -> Result<(), Box> { + let app = Router::new() + .merge(crate::sts::server::routes()) + .with_state(state); + + if let Some(tls_config) = tls_config { + serve_tls_sts_server(listener, app, tls_config).await?; + } else { + axum::serve(listener, app).await?; + } + Ok(()) +} + +async fn serve_tls_sts_server( + listener: tokio::net::TcpListener, + app: Router, + tls_config: Arc, +) -> Result<(), Box> { + let acceptor = TlsAcceptor::from(tls_config); + + loop { + let (tcp_stream, remote_addr) = listener.accept().await?; + let acceptor = acceptor.clone(); + let service = app.clone(); + + tokio::spawn(async move { + let tls_stream = match acceptor.accept(tcp_stream).await { + Ok(stream) => stream, + Err(error) => { + warn!( + %remote_addr, + %error, + "Operator STS TLS handshake failed" + ); + return; + } + }; + + let io = TokioIo::new(tls_stream); + let tower_service = + service.map_request(|request: http::Request| request.map(Body::new)); + let hyper_service = TowerToHyperService::new(tower_service); + + if let Err(error) = HyperBuilder::new(TokioExecutor::new()) + .serve_connection_with_upgrades(io, hyper_service) + .await + { + warn!( + %remote_addr, + %error, + "Operator STS HTTPS connection failed" + ); + } + }); + } +} + fn cert_manager_certificate_gvk() -> GroupVersionKind { GroupVersionKind::gvk( CERT_MANAGER_GROUP, @@ -206,6 +352,12 @@ fn push_unique_tenant_ref(refs: &mut Vec>, tenant_ref: ObjectR } } +pub fn render_crds_yaml() -> Result { + let tenant = serde_yaml_ng::to_string(&Tenant::crd())?; + let policy_binding = serde_yaml_ng::to_string(&PolicyBinding::crd())?; + Ok(format!("{tenant}---\n{policy_binding}")) +} + pub async fn crd(file: Option) -> Result<(), Box> { let mut writer: Pin> = if let Some(file) = file { Box::pin( @@ -220,9 +372,8 @@ pub async fn crd(file: Option) -> Result<(), Box> Box::pin(tokio::io::stdout()) }; - writer - .write_all(serde_yaml_ng::to_string(&Tenant::crd())?.as_bytes()) - .await?; + let yaml = render_crds_yaml()?; + writer.write_all(yaml.as_bytes()).await?; Ok(()) } @@ -303,6 +454,22 @@ mod controller_watch_tests { assert_single_ref(&refs, "tenant-d", "storage"); } + #[test] + fn crd_output_includes_tenant_and_policy_binding_documents() { + let yaml = render_crds_yaml().expect("CRDs render to YAML"); + let documents = yaml + .split("---") + .map(str::trim) + .filter(|document| !document.is_empty()) + .collect::>(); + + assert_eq!(documents.len(), 2); + assert!(documents[0].contains("name: tenants.rustfs.com")); + assert!(documents[1].contains("name: policybindings.sts.rustfs.com")); + assert!(documents[1].contains("kind: PolicyBinding")); + assert!(documents[1].contains("scope: Namespaced")); + } + fn tenant_owner_ref(name: &str) -> metav1::OwnerReference { metav1::OwnerReference { api_version: "rustfs.com/v1alpha1".to_string(), diff --git a/src/sts/binding.rs b/src/sts/binding.rs new file mode 100644 index 0000000..e40d488 --- /dev/null +++ b/src/sts/binding.rs @@ -0,0 +1,148 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::PolicyBinding; + +/// Select policy bindings that match a namespace/service account pair. +pub fn find_matching_bindings( + policy_bindings: &[PolicyBinding], + namespace: &str, + service_account: &str, +) -> Vec { + policy_bindings + .iter() + .filter(|policy_binding| { + policy_binding.spec.application.namespace == namespace + && policy_binding.spec.application.serviceaccount == service_account + }) + .cloned() + .collect() +} + +#[cfg(test)] +mod tests { + use crate::types::v1alpha1::policy_binding::{ + PolicyBinding, PolicyBindingApplication, PolicyBindingSpec, + }; + + #[test] + fn match_policy_bindings_by_serviceaccount_and_namespace() { + let bindings = vec![ + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "build-sa".to_string(), + }, + policies: vec!["policy-a".to_string()], + }, + status: None, + }, + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-b".to_string(), + serviceaccount: "ci-sa".to_string(), + }, + policies: vec!["policy-b".to_string()], + }, + status: None, + }, + ]; + + let matches = super::find_matching_bindings(&bindings, "tenant-a", "build-sa"); + + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].spec.application.namespace, "tenant-a"); + assert_eq!(matches[0].spec.application.serviceaccount, "build-sa"); + } + + #[test] + fn no_match_if_namespace_or_serviceaccount_differs() { + let bindings = vec![ + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "build-sa".to_string(), + }, + policies: vec!["policy-a".to_string()], + }, + status: None, + }, + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "ops-sa".to_string(), + }, + policies: vec!["policy-b".to_string()], + }, + status: None, + }, + ]; + + assert!(super::find_matching_bindings(&bindings, "tenant-b", "build-sa").is_empty()); + assert!(super::find_matching_bindings(&bindings, "tenant-a", "admin-sa").is_empty()); + } + + #[test] + fn match_returns_all_bindings_for_same_namespace_and_service_account() { + let bindings = vec![ + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "sa-a".to_string(), + }, + policies: vec!["policy-1".to_string()], + }, + status: None, + }, + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "sa-a".to_string(), + }, + policies: vec!["policy-2".to_string()], + }, + status: None, + }, + PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "sa-b".to_string(), + }, + policies: vec!["policy-3".to_string()], + }, + status: None, + }, + ]; + + let matches = super::find_matching_bindings(&bindings, "tenant-a", "sa-a"); + + assert_eq!(matches.len(), 2); + assert_eq!(matches[0].spec.policies[0], "policy-1"); + assert_eq!(matches[1].spec.policies[0], "policy-2"); + } +} diff --git a/src/sts/error.rs b/src/sts/error.rs new file mode 100644 index 0000000..4446cda --- /dev/null +++ b/src/sts/error.rs @@ -0,0 +1,108 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// AWS STS XML namespace for Operator-facing payloads. +pub const STS_XML_NAMESPACE: &str = "https://sts.amazonaws.com/doc/2011-06-15/"; + +/// Placeholder request id for deterministic test payloads. +pub const STS_REQUEST_ID: &str = "00000000-0000-0000-0000-000000000000"; + +/// Validation and stub errors for STS request handling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StsError { + MissingParameter { parameter: &'static str }, + InvalidParameterValue { parameter: &'static str }, + InvalidIdentityToken, + AccessDenied, + TenantTlsClientCertificateUnsupported, + InternalError, + NotImplemented, + MalformedPolicyDocument, + PackedPolicyTooLarge, +} + +impl StsError { + /// STS-style short error code. + pub fn code(&self) -> &'static str { + match self { + Self::MissingParameter { .. } => "MissingParameter", + Self::InvalidParameterValue { .. } => "InvalidParameterValue", + Self::InvalidIdentityToken => "InvalidIdentityToken", + Self::AccessDenied => "AccessDenied", + Self::TenantTlsClientCertificateUnsupported => "TenantTlsClientCertificateUnsupported", + Self::InternalError => "InternalError", + Self::NotImplemented => "NotImplemented", + Self::MalformedPolicyDocument => "MalformedPolicyDocument", + Self::PackedPolicyTooLarge => "PackedPolicyTooLarge", + } + } + + /// STS-style error message. + pub fn message(&self) -> String { + match self { + Self::MissingParameter { parameter } => { + format!("Missing required request body parameter {}.", parameter) + } + Self::InvalidParameterValue { parameter } => { + format!("Invalid value for parameter {}.", parameter) + } + Self::InvalidIdentityToken => "The provided web identity token is invalid.".to_string(), + Self::AccessDenied => { + "No matching policy binding was found for this identity.".to_string() + } + Self::TenantTlsClientCertificateUnsupported => { + "Operator STS does not support Tenants that require TLS client certificates." + .to_string() + } + Self::InternalError => "Internal server error.".to_string(), + Self::NotImplemented => "This operation is not yet implemented.".to_string(), + Self::MalformedPolicyDocument => "The policy document is malformed.".to_string(), + Self::PackedPolicyTooLarge => { + "The session policy is too long and must be less than or equal to 2048 bytes." + .to_string() + } + } + } + + /// Render a full AWS STS error response document. + pub fn as_xml(&self) -> String { + render_sts_error_xml(self.code(), &self.message()) + } +} + +/// Escape XML character data and attributes. +pub fn escape_xml(value: &str) -> String { + let mut escaped = String::with_capacity(value.len() + 16); + for ch in value.chars() { + match ch { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + _ => escaped.push(ch), + } + } + escaped +} + +pub fn render_sts_error_xml(code: &str, message: &str) -> String { + let message = escape_xml(message); + let request_id = STS_REQUEST_ID; + + format!( + "Sender{code}{message}{request_id}", + ns = STS_XML_NAMESPACE, + ) +} diff --git a/src/sts/mod.rs b/src/sts/mod.rs new file mode 100644 index 0000000..f5b6635 --- /dev/null +++ b/src/sts/mod.rs @@ -0,0 +1,22 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod binding; +pub mod error; +pub mod rustfs_client; +pub mod server; +pub mod session_policy; +pub mod tls; +pub mod token_review; +pub mod types; diff --git a/src/sts/rustfs_client.rs b/src/sts/rustfs_client.rs new file mode 100644 index 0000000..2bbe6a1 --- /dev/null +++ b/src/sts/rustfs_client.rs @@ -0,0 +1,955 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; + +use hmac::{Hmac, Mac}; +use k8s_openapi::{ByteString, api::core::v1 as corev1}; +use kube::{Api, Client}; +use reqwest::{Certificate, Client as HttpClient, StatusCode}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use url::Url; +use url::form_urlencoded; + +use crate::Tenant; +use crate::sts::types::StsAssumeRoleCredentials; + +const FORM_CONTENT_TYPE: &str = "application/x-www-form-urlencoded"; +const JSON_CONTENT_TYPE: &str = "application/json"; +const ASSUME_ROLE_PATH: &str = "/"; +const ADD_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/add-canned-policy"; +const INFO_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/info-canned-policy"; +const ADMIN_SIGNING_SERVICE: &str = "s3"; +const STS_SIGNING_SERVICE: &str = "sts"; + +/// Credentials read from Tenant `.spec.credsSecret`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RustfsCredentials { + pub access_key: String, + pub secret_key: String, +} + +/// Error type for RustFS admin/STS client operations. +#[derive(Debug)] +pub enum RustfsClientError { + MissingTenantNamespace, + MissingCredsSecret, + MissingCredentialKey { key: &'static str }, + EmptyCredentialValue { key: &'static str }, + InvalidCredentialValue { key: &'static str }, + TenantSecretLookupFailed, + InvalidPolicyName, + InvalidPolicyDocument, + TenantTlsRequired, + TenantTlsNotReady, + TenantTlsClientCertificateRequired, + MissingTenantTlsCaKey { secret: String, key: String }, + TenantTlsCaSecretLookupFailed { secret: String }, + InvalidTenantTlsCa, + TlsClientBuildFailed, + RequestBuildFailed, + RequestFailed, + UnexpectedStatus(StatusCode), + ParseResponseFailed, + SigningFailed, +} + +impl std::fmt::Display for RustfsClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingTenantNamespace => write!(f, "tenant namespace is missing"), + Self::MissingCredsSecret => write!(f, "tenant credsSecret is missing"), + Self::MissingCredentialKey { key } => write!(f, "secret key missing: {key}"), + Self::EmptyCredentialValue { key } => write!(f, "secret key empty: {key}"), + Self::InvalidCredentialValue { key } => { + write!(f, "secret key is not valid utf8: {key}") + } + Self::TenantSecretLookupFailed => { + write!(f, "failed to load tenant credential secret") + } + Self::InvalidPolicyName => write!(f, "invalid policy name"), + Self::InvalidPolicyDocument => write!(f, "failed to parse canned policy response"), + Self::TenantTlsRequired => write!(f, "STS requires a TLS-enabled tenant"), + Self::TenantTlsNotReady => write!(f, "tenant TLS status is not ready"), + Self::TenantTlsClientCertificateRequired => { + write!(f, "tenant TLS requires a client certificate") + } + Self::MissingTenantTlsCaKey { secret, key } => { + write!(f, "tenant TLS CA secret {secret} missing key {key}") + } + Self::TenantTlsCaSecretLookupFailed { secret } => { + write!(f, "failed to load tenant TLS CA secret {secret}") + } + Self::InvalidTenantTlsCa => write!(f, "tenant TLS CA is not a valid PEM bundle"), + Self::TlsClientBuildFailed => write!(f, "failed to build TLS HTTP client"), + Self::RequestBuildFailed => write!(f, "failed to construct request"), + Self::RequestFailed => write!(f, "request failed"), + Self::UnexpectedStatus(status) => write!(f, "upstream returned {status}"), + Self::ParseResponseFailed => write!(f, "failed to parse AssumeRole response"), + Self::SigningFailed => write!(f, "failed to compute request signature"), + } + } +} + +impl std::error::Error for RustfsClientError {} + +#[derive(Debug)] +struct SignedRequest { + amz_date: String, + payload_hash: String, + authorization: String, +} + +/// RustFS admin/STS client. +pub struct RustfsAdminClient { + base_url: String, + access_key: String, + secret_key: String, + region: String, + http_client: HttpClient, +} + +impl RustfsAdminClient { + pub const STS_VERSION: &'static str = "2011-06-15"; + pub const STS_ACTION: &'static str = "AssumeRole"; + + pub fn new_with_base_url( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + ) -> Self { + Self::new_with_base_url_and_http_client(base_url, access_key, secret_key, HttpClient::new()) + } + + pub fn new_with_base_url_and_ca_pem( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + ca_pem: &[u8], + ) -> Result { + let certs = Certificate::from_pem_bundle(ca_pem) + .map_err(|_| RustfsClientError::InvalidTenantTlsCa)?; + let mut builder = HttpClient::builder(); + for cert in certs { + builder = builder.add_root_certificate(cert); + } + let http_client = builder + .build() + .map_err(|_| RustfsClientError::TlsClientBuildFailed)?; + + Ok(Self::new_with_base_url_and_http_client( + base_url, + access_key, + secret_key, + http_client, + )) + } + + pub fn new_with_base_url_and_http_client( + base_url: impl Into, + access_key: impl Into, + secret_key: impl Into, + http_client: HttpClient, + ) -> Self { + Self { + base_url: base_url.into(), + access_key: access_key.into(), + secret_key: secret_key.into(), + region: "us-east-1".to_string(), + http_client, + } + } + + pub fn from_tenant( + tenant: &Tenant, + credentials: RustfsCredentials, + ) -> Result { + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let service_name = tenant + .new_io_service() + .metadata + .name + .unwrap_or_else(|| format!("{}-io", tenant.name())); + + Ok(Self::new_with_base_url( + format!("http://{service_name}.{namespace}.svc:9000"), + credentials.access_key, + credentials.secret_key, + )) + } + + pub async fn from_tls_tenant_for_sts( + kube_client: &Client, + tenant: &Tenant, + credentials: RustfsCredentials, + ) -> Result { + if !tenant_tls_enabled(tenant) { + return Err(RustfsClientError::TenantTlsRequired); + } + if tenant_tls_client_certificate_required(tenant) { + return Err(RustfsClientError::TenantTlsClientCertificateRequired); + } + + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let service_name = tenant + .new_io_service() + .metadata + .name + .unwrap_or_else(|| format!("{}-io", tenant.name())); + let base_url = format!("https://{service_name}.{namespace}.svc:9000"); + + match Self::load_tenant_tls_ca(kube_client, tenant).await? { + Some(ca_pem) => Self::new_with_base_url_and_ca_pem( + base_url, + credentials.access_key, + credentials.secret_key, + &ca_pem, + ), + None => Ok(Self::new_with_base_url( + base_url, + credentials.access_key, + credentials.secret_key, + )), + } + } + + pub async fn load_tenant_tls_ca( + kube_client: &Client, + tenant: &Tenant, + ) -> Result>, RustfsClientError> { + if !tenant_tls_enabled(tenant) { + return Ok(None); + } + + let tls_status = tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .filter(|tls| tls.ready) + .ok_or(RustfsClientError::TenantTlsNotReady)?; + + let Some(ca_ref) = tls_status.ca_secret_ref.as_ref() else { + return Ok(None); + }; + + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let api: Api = Api::namespaced(kube_client.clone(), &namespace); + let secret = api.get(&ca_ref.name).await.map_err(|_| { + RustfsClientError::TenantTlsCaSecretLookupFailed { + secret: ca_ref.name.clone(), + } + })?; + let key = ca_ref.key.as_deref().unwrap_or("ca.crt"); + let ca_pem = secret + .data + .as_ref() + .and_then(|data| data.get(key)) + .map(|bytes| bytes.0.clone()) + .filter(|bytes| !bytes.is_empty()) + .ok_or_else(|| RustfsClientError::MissingTenantTlsCaKey { + secret: ca_ref.name.clone(), + key: key.to_string(), + })?; + + Ok(Some(ca_pem)) + } + + /// Read Tenant credential Secret and return access/secret key pair. + pub async fn load_tenant_credentials( + kube_client: &Client, + tenant: &Tenant, + ) -> Result { + let reference = tenant + .spec + .creds_secret + .as_ref() + .ok_or(RustfsClientError::MissingCredsSecret)?; + + let namespace = tenant + .namespace() + .map_err(|_| RustfsClientError::MissingTenantNamespace)?; + let api: Api = Api::namespaced(kube_client.clone(), &namespace); + let secret = api + .get(&reference.name) + .await + .map_err(|_| RustfsClientError::TenantSecretLookupFailed)?; + + extract_credentials(secret.data.as_ref()) + } + + /// Query RustFS admin policy endpoint. + pub async fn get_canned_policy(&self, policy_name: &str) -> Result { + if policy_name.trim().is_empty() { + return Err(RustfsClientError::InvalidPolicyName); + } + + let query = build_query_pairs(&[("name", policy_name)]); + let path = INFO_CANNED_POLICY_PATH; + let url = format!("{}{}", self.base_url.trim_end_matches('/'), path); + let url = if query.is_empty() { + url + } else { + format!("{url}?{query}") + }; + + let signed = self.sign_request("GET", path, &query, "", None, ADMIN_SIGNING_SERVICE)?; + let host = self.host()?; + + let response = self + .http_client + .get(url) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if !response.status().is_success() { + return Err(RustfsClientError::UnexpectedStatus(response.status())); + } + + let body = response + .text() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + extract_canned_policy_document(&body) + } + + /// Add or replace a RustFS canned policy through the admin API. + pub async fn add_canned_policy( + &self, + policy_name: &str, + policy_document: &str, + ) -> Result<(), RustfsClientError> { + if policy_name.trim().is_empty() { + return Err(RustfsClientError::InvalidPolicyName); + } + serde_json::from_str::(policy_document) + .map_err(|_| RustfsClientError::InvalidPolicyDocument)?; + + let query = build_query_pairs(&[("name", policy_name)]); + let path = ADD_CANNED_POLICY_PATH; + let url = format!("{}{}?{query}", self.base_url.trim_end_matches('/'), path); + + let signed = self.sign_request( + "PUT", + path, + &query, + policy_document, + Some(JSON_CONTENT_TYPE), + ADMIN_SIGNING_SERVICE, + )?; + let host = self.host()?; + + let response = self + .http_client + .put(url) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .header("content-type", JSON_CONTENT_TYPE) + .body(policy_document.to_string()) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if !response.status().is_success() { + return Err(RustfsClientError::UnexpectedStatus(response.status())); + } + + Ok(()) + } + + /// Send AssumeRole request to RustFS admin STS endpoint (`/`). + pub async fn assume_role( + &self, + policy: Option<&str>, + duration_seconds: u64, + ) -> Result { + let mut params = vec![ + ("Version", Self::STS_VERSION.to_string()), + ("Action", Self::STS_ACTION.to_string()), + ("DurationSeconds", duration_seconds.to_string()), + ]; + + if let Some(policy) = policy { + params.push(("Policy", policy.to_string())); + } + + let body = build_query_pairs( + ¶ms + .iter() + .map(|(k, v)| (&k[..], &v[..])) + .collect::>(), + ); + + let path = ASSUME_ROLE_PATH; + let signed = self.sign_request( + "POST", + path, + "", + &body, + Some(FORM_CONTENT_TYPE), + STS_SIGNING_SERVICE, + )?; + let host = self.host()?; + + let response = self + .http_client + .post(format!("{}/", self.base_url.trim_end_matches('/'))) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .header("content-type", FORM_CONTENT_TYPE) + .body(body) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if !response.status().is_success() { + return Err(RustfsClientError::UnexpectedStatus(response.status())); + } + + let body = response + .text() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + parse_assume_role_response(&body).ok_or(RustfsClientError::ParseResponseFailed) + } + + fn sign_request( + &self, + method: &str, + path: &str, + canonical_query: &str, + payload: &str, + content_type: Option<&str>, + service: &str, + ) -> Result { + let now = chrono::Utc::now(); + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date_stamp = now.format("%Y%m%d").to_string(); + let payload_hash = sha256_hex(payload.as_bytes()); + + let host = self.host()?; + let mut signed_headers = vec![ + ("host", host.as_str()), + ("x-amz-content-sha256", payload_hash.as_str()), + ("x-amz-date", amz_date.as_str()), + ]; + + if let Some(content_type) = content_type { + signed_headers.push(("content-type", content_type)); + } + signed_headers.sort_by_key(|(name, _)| *name); + + let canonical_headers: String = signed_headers + .iter() + .map(|(key, value)| format!("{key}:{value}\n")) + .collect(); + let mut signed_header_names = String::new(); + for (index, (name, _)) in signed_headers.iter().enumerate() { + if index > 0 { + signed_header_names.push(';'); + } + signed_header_names.push_str(name); + } + + let canonical_request = format!( + "{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_header_names}\n{payload_hash}", + ); + + let credential_scope = format!("{date_stamp}/{}/{service}/aws4_request", self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{}", + sha256_hex(canonical_request.as_bytes()) + ); + + let signing_key = derive_signing_key(&self.secret_key, &date_stamp, &self.region, service)?; + let signature = hmac_sha256_hex(&signing_key, &string_to_sign)?; + let authorization = format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + self.access_key, credential_scope, signed_header_names, signature + ); + + Ok(SignedRequest { + amz_date, + payload_hash, + authorization, + }) + } + + fn host(&self) -> Result { + let parsed = + Url::parse(&self.base_url).map_err(|_| RustfsClientError::RequestBuildFailed)?; + let mut host = parsed + .host_str() + .map(str::to_string) + .or_else(|| parsed.host().map(|h| h.to_string())) + .ok_or(RustfsClientError::RequestBuildFailed)?; + if let Some(port) = parsed.port() { + host.push(':'); + host.push_str(&port.to_string()); + } + Ok(host) + } +} + +fn extract_credentials( + data: Option<&BTreeMap>, +) -> Result { + let secret_data = data.ok_or(RustfsClientError::TenantSecretLookupFailed)?; + + Ok(RustfsCredentials { + access_key: get_secret_value(secret_data, "accesskey")?, + secret_key: get_secret_value(secret_data, "secretkey")?, + }) +} + +fn tenant_tls_enabled(tenant: &Tenant) -> bool { + tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) +} + +fn tenant_tls_client_certificate_required(tenant: &Tenant) -> bool { + tenant + .status + .as_ref() + .and_then(|status| status.certificates.tls.as_ref()) + .and_then(|tls| tls.client_ca_secret_ref.as_ref()) + .is_some() +} + +fn get_secret_value( + data: &BTreeMap, + field: &'static str, +) -> Result { + let raw = data + .get(field) + .ok_or(RustfsClientError::MissingCredentialKey { key: field })?; + + let value = String::from_utf8(raw.0.clone()) + .map_err(|_| RustfsClientError::InvalidCredentialValue { key: field })?; + + if value.is_empty() { + return Err(RustfsClientError::EmptyCredentialValue { key: field }); + } + + Ok(value) +} + +fn build_query_pairs(params: &[(&str, &str)]) -> String { + let mut pairs: Vec<(String, String)> = params + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + pairs.sort_by(|(k1, v1), (k2, v2)| k1.cmp(k2).then(v1.cmp(v2))); + + let mut serializer = form_urlencoded::Serializer::new(String::new()); + for (key, value) in pairs { + serializer.append_pair(&key, &value); + } + + serializer.finish() +} + +fn extract_canned_policy_document(body: &str) -> Result { + let value = serde_json::from_str::(body) + .map_err(|_| RustfsClientError::InvalidPolicyDocument)?; + let policy = value.get("policy").unwrap_or(&value); + + serde_json::to_string(policy).map_err(|_| RustfsClientError::InvalidPolicyDocument) +} + +fn sha256_hex(payload: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(payload); + hex::encode(hasher.finalize()) +} + +fn hmac_sha256(key: &[u8], message: &str) -> Result, RustfsClientError> { + let mut mac = + Hmac::::new_from_slice(key).map_err(|_| RustfsClientError::SigningFailed)?; + mac.update(message.as_bytes()); + Ok(mac.finalize().into_bytes().to_vec()) +} + +fn hmac_sha256_hex(key: &[u8], message: &str) -> Result { + let bytes = hmac_sha256(key, message)?; + Ok(hex::encode(bytes)) +} + +fn derive_signing_key( + secret_key: &str, + date_stamp: &str, + region: &str, + service: &str, +) -> Result, RustfsClientError> { + let k_secret = format!("AWS4{secret_key}").into_bytes(); + let k_date = hmac_sha256(&k_secret, date_stamp)?; + let k_region = hmac_sha256(&k_date, region)?; + let k_service = hmac_sha256(&k_region, service)?; + hmac_sha256(&k_service, "aws4_request") +} + +fn parse_assume_role_response(body: &str) -> Option { + let access_key_id = extract_xml_tag(body, "AccessKeyId")?; + let secret_access_key = extract_xml_tag(body, "SecretAccessKey")?; + let session_token = extract_xml_tag(body, "SessionToken")?; + let expiration = extract_xml_tag(body, "Expiration")?; + + Some(StsAssumeRoleCredentials { + access_key_id, + secret_access_key, + session_token, + expiration, + }) +} + +fn extract_xml_tag(document: &str, tag: &str) -> Option { + let open = format!("<{tag}>"); + let close = format!(""); + + let open_idx = document.find(&open)?; + let start = open_idx + open.len(); + let rest = &document[start..]; + let end = rest.find(&close)?; + + Some(rest[..end].trim().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + Router, + body::Body, + extract::State, + http::{Request, StatusCode}, + routing::{get, post, put}, + }; + use std::sync::Arc; + use tokio::sync::Mutex; + + fn secret_with_fields(fields: Vec<(&str, &[u8])>) -> corev1::Secret { + let mut data = BTreeMap::new(); + for (key, value) in fields { + data.insert(key.to_string(), ByteString(value.to_vec())); + } + + corev1::Secret { + data: Some(data), + ..Default::default() + } + } + + #[test] + fn extract_credentials_reports_missing_access_key() { + let secret = secret_with_fields(vec![("secretkey", b"sekret")]); + + let err = + extract_credentials(secret.data.as_ref()).expect_err("expected missing access key"); + assert!(matches!( + err, + RustfsClientError::MissingCredentialKey { key: "accesskey" } + )); + } + + #[test] + fn extract_credentials_reports_non_utf8_access_key() { + let secret = + secret_with_fields(vec![("accesskey", &[0xff, 0xfe]), ("secretkey", b"sekret")]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); + assert!(matches!( + err, + RustfsClientError::InvalidCredentialValue { key: "accesskey" } + )); + } + + #[test] + fn extract_credentials_reports_missing_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"access")]); + + let err = + extract_credentials(secret.data.as_ref()).expect_err("expected missing secret key"); + assert!(matches!( + err, + RustfsClientError::MissingCredentialKey { key: "secretkey" } + )); + } + + #[test] + fn extract_credentials_reports_non_utf8_secret_key() { + let secret = + secret_with_fields(vec![("accesskey", b"access"), ("secretkey", &[0xff, 0xfe])]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected invalid utf8"); + assert!(matches!( + err, + RustfsClientError::InvalidCredentialValue { key: "secretkey" } + )); + } + + #[test] + fn extract_credentials_reports_empty_secret_key() { + let secret = secret_with_fields(vec![("accesskey", b"abc"), ("secretkey", b"")]); + + let err = extract_credentials(secret.data.as_ref()).expect_err("expected empty secret key"); + assert!(matches!( + err, + RustfsClientError::EmptyCredentialValue { key: "secretkey" } + )); + } + + #[test] + fn parse_assume_role_xml_success_and_failure() { + let body_ok = "AKISECTOKEN2026-01-01T00:00:00Z"; + let parsed = + parse_assume_role_response(body_ok).expect("valid assume role response should parse"); + + assert_eq!(parsed.access_key_id, "AKI"); + assert_eq!(parsed.secret_access_key, "SEC"); + assert_eq!(parsed.session_token, "TOKEN"); + assert_eq!(parsed.expiration, "2026-01-01T00:00:00Z"); + + assert!(parse_assume_role_response("").is_none()); + } + + #[derive(Clone, Default)] + struct Capture { + path: Arc>, + query: Arc>, + body: Arc>, + authorization: Arc>, + } + + #[tokio::test] + async fn assume_role_request_targets_root_path_and_action_is_assume_role() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new().route( + "/", + post( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.body.lock().await = body; + *c.authorization.lock().await = authorization; + + let response = + "AKISECTOKEN2026-01-01T00:00:00Z"; + (StatusCode::OK, response) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let creds = client + .assume_role(Some("{\"Statement\": []}"), 3600) + .await + .unwrap(); + assert_eq!(creds.access_key_id, "AKI"); + + assert_eq!(&*capture.path.lock().await, "/"); + assert!(capture.body.lock().await.contains("Action=AssumeRole")); + assert!(capture.body.lock().await.contains("Version=2011-06-15")); + assert!(capture.body.lock().await.contains("DurationSeconds=3600")); + assert!(capture.query.lock().await.is_empty()); + assert!( + capture + .authorization + .lock() + .await + .contains("/sts/aws4_request") + ); + + server.abort(); + } + + #[tokio::test] + async fn info_canned_policy_uses_expected_path_and_query() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/rustfs/admin/v3/info-canned-policy", + get( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.authorization.lock().await = authorization; + + ( + StatusCode::OK, + "{\"policy_name\":\"tenant-policy\",\"policy\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"allow\",\"Effect\":\"Allow\"}]}}", + ) + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + + let policy = client.get_canned_policy("tenant-policy").await.unwrap(); + let policy_value = serde_json::from_str::(&policy).unwrap(); + assert_eq!(policy_value["Version"], "2012-10-17"); + assert_eq!(policy_value["Statement"][0]["Sid"], "allow"); + + assert_eq!( + &*capture.path.lock().await, + "/rustfs/admin/v3/info-canned-policy" + ); + assert!(capture.query.lock().await.contains("name=tenant-policy")); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); + } + + #[tokio::test] + async fn add_canned_policy_uses_expected_path_query_body_and_admin_signing() { + let capture = Capture::default(); + let route_capture = capture.clone(); + + let router = Router::new() + .route( + "/rustfs/admin/v3/add-canned-policy", + put( + move |State(c): State, req: Request| async move { + let path = req.uri().path().to_string(); + let query = req.uri().query().unwrap_or("").to_string(); + let authorization = req + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(body_bytes.to_vec()).unwrap(); + + *c.path.lock().await = path; + *c.query.lock().await = query; + *c.authorization.lock().await = authorization; + *c.body.lock().await = body; + + StatusCode::OK + }, + ), + ) + .with_state(route_capture.clone()); + + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let policy = r#"{"Version":"2012-10-17","Statement":[]}"#; + + client + .add_canned_policy("tenant-policy", policy) + .await + .unwrap(); + + assert_eq!( + &*capture.path.lock().await, + "/rustfs/admin/v3/add-canned-policy" + ); + assert!(capture.query.lock().await.contains("name=tenant-policy")); + assert_eq!(&*capture.body.lock().await, policy); + assert!( + capture + .authorization + .lock() + .await + .contains("/s3/aws4_request") + ); + + server.abort(); + } + + #[test] + fn extract_canned_policy_document_accepts_raw_policy_document() { + let raw_policy = + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"raw\",\"Effect\":\"Allow\"}]}"; + + let policy = extract_canned_policy_document(raw_policy).unwrap(); + + let policy_value = serde_json::from_str::(&policy).unwrap(); + assert_eq!(policy_value["Version"], "2012-10-17"); + assert_eq!(policy_value["Statement"][0]["Sid"], "raw"); + } +} diff --git a/src/sts/server.rs b/src/sts/server.rs new file mode 100644 index 0000000..85bbdf8 --- /dev/null +++ b/src/sts/server.rs @@ -0,0 +1,1094 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use async_trait::async_trait; +use axum::{ + Router, + extract::{Form, Path, State}, + http::{StatusCode, header}, + response::{IntoResponse, Response}, + routing::post, +}; +use k8s_openapi::api::authentication::v1::{TokenReview, TokenReviewSpec}; +use kube::{Api, Client, api::ListParams}; + +use crate::console::state::AppState; +use crate::sts::binding; +use crate::sts::error::StsError; +use crate::sts::rustfs_client::{RustfsAdminClient, RustfsClientError}; +use crate::sts::session_policy; +use crate::sts::token_review::{self, TokenReviewError}; +use crate::sts::types::{ + AssumeRoleWithWebIdentityForm, StsParsedRequest, StsWebIdentityResponseContext, parse_sts_form, + render_assume_role_with_web_identity_response, render_not_implemented_response, +}; +use crate::{PolicyBinding, Tenant}; + +const DEFAULT_STS_AUDIENCE: &str = "sts.rustfs.com"; + +/// Build STS routes mounted at root path `/sts`. +pub fn routes() -> Router { + Router::new().route( + "/sts/:tenant_namespace/:tenant_name", + post(assume_role_with_web_identity_for_tenant), + ) +} + +/// Handle POST /sts/{tenantNamespace}/{tenantName}. +async fn assume_role_with_web_identity_for_tenant( + State(state): State, + Path((tenant_namespace, tenant_name)): Path<(String, String)>, + Form(form): Form, +) -> Response { + assume_role_with_web_identity(state, tenant_namespace, tenant_name, form).await +} + +async fn assume_role_with_web_identity( + state: AppState, + tenant_namespace: String, + tenant_name: String, + form: AssumeRoleWithWebIdentityForm, +) -> Response { + let parsed_request = match parse_sts_form(tenant_namespace, tenant_name, form) { + Ok(parsed_request) => parsed_request, + Err(error) => { + tracing::warn!( + error_code = %error.code(), + "STS request failed request validation" + ); + return xml_response(StatusCode::BAD_REQUEST, error.as_xml()); + } + }; + + let Some(kube_client) = state.kube_client.as_ref() else { + tracing::info!( + tenant_namespace = %parsed_request.tenant_namespace, + "STS request accepted by compatibility mode: kube_client is not available" + ); + return xml_response( + StatusCode::NOT_IMPLEMENTED, + render_not_implemented_response(), + ); + }; + + let runtime = RealStsRuntime { + kube_client: kube_client.clone(), + }; + + process_assume_role_request(&runtime, parsed_request).await +} + +#[async_trait] +trait StsRuntime { + async fn authenticate_service_account( + &self, + token: &str, + audience: &str, + ) -> Result; + + async fn list_policy_bindings( + &self, + tenant_namespace: &str, + ) -> Result, StsError>; + + async fn select_tenant( + &self, + tenant_namespace: &str, + tenant_name: &str, + ) -> Result; + + async fn create_rustfs_admin_client( + &self, + tenant: &Tenant, + ) -> Result; + + async fn fetch_canned_policy( + &self, + rustfs_client: &RustfsAdminClient, + policy_name: &str, + ) -> Result; + + async fn assume_role( + &self, + rustfs_client: &RustfsAdminClient, + policy: Option<&str>, + duration_seconds: u64, + ) -> Result; +} + +struct RealStsRuntime { + kube_client: Client, +} + +#[async_trait] +impl StsRuntime for RealStsRuntime { + async fn authenticate_service_account( + &self, + token: &str, + audience: &str, + ) -> Result { + authenticate_service_account(&self.kube_client, token, audience).await + } + + async fn list_policy_bindings( + &self, + tenant_namespace: &str, + ) -> Result, StsError> { + list_policy_bindings(&self.kube_client, tenant_namespace).await + } + + async fn select_tenant( + &self, + tenant_namespace: &str, + tenant_name: &str, + ) -> Result { + select_tenant(&self.kube_client, tenant_namespace, tenant_name).await + } + + async fn create_rustfs_admin_client( + &self, + tenant: &Tenant, + ) -> Result { + create_rustfs_admin_client(&self.kube_client, tenant).await + } + + async fn fetch_canned_policy( + &self, + rustfs_client: &RustfsAdminClient, + policy_name: &str, + ) -> Result { + rustfs_client.get_canned_policy(policy_name).await + } + + async fn assume_role( + &self, + rustfs_client: &RustfsAdminClient, + policy: Option<&str>, + duration_seconds: u64, + ) -> Result { + rustfs_client.assume_role(policy, duration_seconds).await + } +} + +async fn process_assume_role_request( + runtime: &(impl StsRuntime + Send + Sync), + parsed_request: StsParsedRequest, +) -> Response { + let sts_audience = operator_sts_audience(); + let identity = match runtime + .authenticate_service_account(&parsed_request.web_identity_token, &sts_audience) + .await + { + Ok(identity) => identity, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error.code(), + "TokenReview denied STS request" + ); + return xml_response(StatusCode::BAD_REQUEST, error.as_xml()); + } + }; + + let policy_bindings = match runtime + .list_policy_bindings(&parsed_request.tenant_namespace) + .await + { + Ok(policy_bindings) => policy_bindings, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error.code(), + "Failed listing PolicyBindings for STS authorization" + ); + return xml_response(StatusCode::BAD_REQUEST, error.as_xml()); + } + }; + + let matching_bindings = binding::find_matching_bindings( + &policy_bindings, + &identity.namespace, + &identity.service_account, + ); + + if matching_bindings.is_empty() { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + service_account_namespace = %identity.namespace, + service_account = %identity.service_account, + "No PolicyBinding matched service account for this STS request" + ); + return xml_response(StatusCode::FORBIDDEN, StsError::AccessDenied.as_xml()); + } + + let tenant = match runtime + .select_tenant( + &parsed_request.tenant_namespace, + &parsed_request.tenant_name, + ) + .await + { + Ok(tenant) => tenant, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error.code(), + "Failed selecting tenant for STS request" + ); + return xml_response(StatusCode::BAD_REQUEST, error.as_xml()); + } + }; + + let rustfs_client = match runtime.create_rustfs_admin_client(&tenant).await { + Ok(rustfs_client) => rustfs_client, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error.code(), + "Failed creating RustFS admin client" + ); + return xml_response(sts_error_status(&error), error.as_xml()); + } + }; + + let binding_policies = + match resolve_binding_policies(runtime, &rustfs_client, &matching_bindings).await { + Ok(binding_policies) => binding_policies, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error.code(), + "Failed resolving PolicyBinding policy documents" + ); + let status = match error { + StsError::InternalError => StatusCode::INTERNAL_SERVER_ERROR, + StsError::AccessDenied => StatusCode::FORBIDDEN, + _ => StatusCode::BAD_REQUEST, + }; + return xml_response(status, error.as_xml()); + } + }; + + let merged_policy = match session_policy::merge_session_policies( + parsed_request.policy.as_deref(), + &binding_policies, + ) { + Ok(policy) => policy, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error.code(), + "Failed to build merged STS session policy" + ); + return xml_response(StatusCode::BAD_REQUEST, error.as_xml()); + } + }; + + let credentials = match runtime + .assume_role( + &rustfs_client, + merged_policy.as_deref(), + parsed_request.duration_seconds, + ) + .await + { + Ok(credentials) => credentials, + Err(error) => { + tracing::warn!( + tenant_namespace = %parsed_request.tenant_namespace, + error = %error, + "Failed calling RustFS AssumeRole" + ); + return xml_response( + StatusCode::INTERNAL_SERVER_ERROR, + StsError::InternalError.as_xml(), + ); + } + }; + + let tenant_name = tenant + .metadata + .name + .clone() + .unwrap_or_else(|| parsed_request.tenant_name.clone()); + + let binding_policy_count: usize = matching_bindings + .iter() + .map(|binding| binding.spec.policies.len()) + .sum(); + + tracing::info!( + tenant = %tenant_name, + tenant_namespace = %parsed_request.tenant_namespace, + service_account_namespace = %identity.namespace, + service_account = %identity.service_account, + action = %parsed_request.action, + duration_seconds = parsed_request.duration_seconds, + version = %parsed_request.version, + request_policy = parsed_request.policy.is_some(), + binding_policy_count, + "STS request passed TokenReview + PolicyBinding authorization checks" + ); + + xml_response( + StatusCode::OK, + render_assume_role_with_web_identity_response( + &credentials, + &web_identity_response_context( + &parsed_request, + &identity, + &sts_audience, + &tenant_name, + merged_policy.as_deref(), + &credentials, + ), + ), + ) +} + +fn web_identity_response_context( + parsed_request: &StsParsedRequest, + identity: &token_review::ServiceAccountIdentity, + audience: &str, + tenant_name: &str, + merged_policy: Option<&str>, + credentials: &crate::sts::types::StsAssumeRoleCredentials, +) -> StsWebIdentityResponseContext { + let subject = format!( + "system:serviceaccount:{}:{}", + identity.namespace, identity.service_account + ); + let role_session_name = format!("{}:{}", identity.namespace, identity.service_account); + let packed_policy_size = merged_policy + .map(|policy| { + let size = policy.len() * 100; + size.div_ceil(session_policy::MAX_SESSION_POLICY_SIZE) + .min(100) as u8 + }) + .unwrap_or(0); + + StsWebIdentityResponseContext { + subject, + audience: audience.to_string(), + provider: "kubernetes".to_string(), + assumed_role_arn: format!( + "arn:rustfs:sts::{}:assumed-role/{}/{}", + parsed_request.tenant_namespace, tenant_name, role_session_name + ), + assumed_role_id: format!("{}:{}", credentials.access_key_id, role_session_name), + packed_policy_size, + } +} + +fn operator_sts_audience() -> String { + std::env::var("OPERATOR_STS_AUDIENCE") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_STS_AUDIENCE.to_string()) +} + +async fn resolve_binding_policies( + runtime: &R, + rustfs_client: &RustfsAdminClient, + matching_bindings: &[PolicyBinding], +) -> Result, StsError> { + let mut policies = Vec::new(); + let mut referenced_policy_count = 0usize; + + for binding in matching_bindings { + if binding.spec.policies.is_empty() { + tracing::warn!( + policy_binding = binding.metadata.name.as_deref().unwrap_or(""), + "PolicyBinding matched service account but does not reference any policies" + ); + } + + for policy_name in &binding.spec.policies { + referenced_policy_count += 1; + + let raw_policy = match runtime + .fetch_canned_policy(rustfs_client, policy_name) + .await + { + Ok(policy) => policy, + Err(error) => { + tracing::warn!( + policy = %policy_name, + error = %error, + "Failed fetching PolicyBinding policy; skipping policy" + ); + continue; + } + }; + + match session_policy::normalize_policy_for_merge(&raw_policy) { + Ok(compact) => policies.push(compact), + Err(error) => { + tracing::warn!( + policy = %policy_name, + error = %error.code(), + "Invalid PolicyBinding policy document; skipping policy" + ); + continue; + } + } + } + } + + if policies.is_empty() { + tracing::warn!( + matched_binding_count = matching_bindings.len(), + referenced_policy_count, + "No valid PolicyBinding policy documents were resolved" + ); + return Err(StsError::AccessDenied); + } + + Ok(policies) +} + +async fn authenticate_service_account( + client: &Client, + token: &str, + audience: &str, +) -> Result { + let request = TokenReview { + metadata: Default::default(), + spec: TokenReviewSpec { + audiences: Some(vec![audience.to_string()]), + token: Some(token.to_string()), + }, + status: None, + }; + + let api: Api = Api::all(client.clone()); + let token_review = api + .create(&kube::api::PostParams::default(), &request) + .await + .map_err(|_| StsError::InternalError)?; + + let status = token_review + .status + .as_ref() + .ok_or(StsError::InvalidIdentityToken)?; + + token_review::extract_service_account_identity_for_audience(status, Some(audience)) + .map_err(map_token_review_error) +} + +fn map_token_review_error(error: TokenReviewError) -> StsError { + match error { + TokenReviewError::MissingTokenReview => StsError::InvalidIdentityToken, + TokenReviewError::NotAuthenticated => StsError::InvalidIdentityToken, + TokenReviewError::MissingAudience => StsError::InvalidIdentityToken, + TokenReviewError::InvalidAudience => StsError::InvalidIdentityToken, + TokenReviewError::MissingUsername => StsError::InvalidIdentityToken, + TokenReviewError::InvalidUsername => StsError::InvalidIdentityToken, + TokenReviewError::InvalidUsernameFormat => StsError::InvalidIdentityToken, + } +} + +async fn list_policy_bindings( + client: &Client, + tenant_namespace: &str, +) -> Result, StsError> { + let api: Api = Api::namespaced(client.clone(), tenant_namespace); + api.list(&ListParams::default()) + .await + .map(|list| list.items) + .map_err(|_| StsError::InternalError) +} + +async fn select_tenant( + client: &Client, + tenant_namespace: &str, + tenant_name: &str, +) -> Result { + let api: Api = Api::namespaced(client.clone(), tenant_namespace); + + api.get(tenant_name).await.map_err(|error| match error { + kube::Error::Api(api_error) if api_error.code == 404 => StsError::InvalidParameterValue { + parameter: "tenantName", + }, + _ => StsError::InternalError, + }) +} + +async fn create_rustfs_admin_client( + client: &Client, + tenant: &Tenant, +) -> Result { + if !tenant.spec.tls.as_ref().is_some_and(|tls| tls.is_enabled()) { + return Err(StsError::InvalidParameterValue { + parameter: "tenantTls", + }); + } + + let credentials = RustfsAdminClient::load_tenant_credentials(client, tenant) + .await + .map_err(|_| StsError::InternalError)?; + + RustfsAdminClient::from_tls_tenant_for_sts(client, tenant, credentials) + .await + .map_err(map_rustfs_client_creation_error) +} + +fn map_rustfs_client_creation_error(error: RustfsClientError) -> StsError { + match error { + RustfsClientError::TenantTlsRequired => StsError::InvalidParameterValue { + parameter: "tenantTls", + }, + RustfsClientError::TenantTlsClientCertificateRequired => { + StsError::TenantTlsClientCertificateUnsupported + } + _ => StsError::InternalError, + } +} + +fn xml_response(status_code: StatusCode, body: String) -> Response { + ( + status_code, + [(header::CONTENT_TYPE, "application/xml")], + body, + ) + .into_response() +} + +fn sts_error_status(error: &StsError) -> StatusCode { + match error { + StsError::AccessDenied => StatusCode::FORBIDDEN, + StsError::InvalidParameterValue { .. } + | StsError::TenantTlsClientCertificateUnsupported + | StsError::MissingParameter { .. } + | StsError::MalformedPolicyDocument + | StsError::PackedPolicyTooLarge + | StsError::InvalidIdentityToken => StatusCode::BAD_REQUEST, + StsError::NotImplemented => StatusCode::NOT_IMPLEMENTED, + StsError::InternalError => StatusCode::INTERNAL_SERVER_ERROR, + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::sync::Mutex; + + use super::*; + use crate::sts::types::{STS_API_VERSION, STS_WEB_IDENTITY_ACTION, StsAssumeRoleCredentials}; + use crate::types::v1alpha1::policy_binding::{PolicyBindingApplication, PolicyBindingSpec}; + use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, + }; + use tower::ServiceExt; + + #[tokio::test] + async fn namespace_only_route_is_not_registered() { + let app = routes().with_state(AppState::new("test-secret".to_string())); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/sts/tenant-a") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from( + "Version=2011-06-15&Action=AssumeRoleWithWebIdentity&WebIdentityToken=abc", + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn valid_explicit_tenant_route_returns_not_implemented_if_runtime_is_unavailable() { + let app = routes().with_state(AppState::new("test-secret".to_string())); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/sts/tenant-a/rustfs-a") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from( + "Version=2011-06-15&Action=AssumeRoleWithWebIdentity&WebIdentityToken=abc", + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); + } + + #[tokio::test] + async fn invalid_request_returns_bad_request_xml_error() { + let app = routes().with_state(AppState::new("test-secret".to_string())); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/sts/tenant-a/rustfs-a") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from( + "Version=2011-06-15&Action=AssumeRoleWithWebIdentity", + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let text = String::from_utf8(body.to_vec()).unwrap(); + assert!(text.contains("MissingParameter")); + } + + #[tokio::test] + async fn sts_handler_rejects_invalid_tokenreview_identity() { + let runtime = MockStsRuntime { + identity: Mutex::new(Some(Err(StsError::InvalidIdentityToken))), + policy_bindings: Mutex::new(Some(Ok(vec![]))), + tenant: Mutex::new(Some(Ok(crate::types::v1alpha1::tenant::Tenant { + metadata: Default::default(), + spec: Default::default(), + status: None, + }))), + create_client: Mutex::new(Some(Ok(RustfsAdminClient::new_with_base_url( + "http://127.0.0.1:1", + "access-key", + "secret-key", + )))), + fetch_policy_results: Mutex::new(VecDeque::new()), + assume_role_result: Mutex::new(Some(Ok(StsAssumeRoleCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string(), + session_token: "token".to_string(), + expiration: "2024-01-01T00:00:00Z".to_string(), + }))), + }; + + let form = AssumeRoleWithWebIdentityForm { + version: Some(STS_API_VERSION.to_string()), + action: Some(STS_WEB_IDENTITY_ACTION.to_string()), + web_identity_token: Some("sa-token".to_string()), + duration_seconds: None, + policy: None, + }; + let parsed = parse_sts_form("tenant-a".to_string(), "rustfs-a".to_string(), form).unwrap(); + + let response = process_assume_role_request(&runtime, parsed).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn sts_handler_requires_matching_policy_binding() { + let runtime = MockStsRuntime { + identity: Mutex::new(Some(Ok(token_review::ServiceAccountIdentity { + namespace: "tenant-a".to_string(), + service_account: "workload-sa".to_string(), + }))), + policy_bindings: Mutex::new(Some(Ok(vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "other-ns".to_string(), + serviceaccount: "other-sa".to_string(), + }, + policies: vec!["policy-a".to_string()], + }, + status: None, + }]))), + tenant: Mutex::new(Some(Ok(crate::types::v1alpha1::tenant::Tenant { + metadata: Default::default(), + spec: Default::default(), + status: None, + }))), + create_client: Mutex::new(Some(Ok(RustfsAdminClient::new_with_base_url( + "http://127.0.0.1:1", + "access-key", + "secret-key", + )))), + fetch_policy_results: Mutex::new(VecDeque::new()), + assume_role_result: Mutex::new(Some(Ok(StsAssumeRoleCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string(), + session_token: "token".to_string(), + expiration: "2024-01-01T00:00:00Z".to_string(), + }))), + }; + + let form = AssumeRoleWithWebIdentityForm { + version: Some(STS_API_VERSION.to_string()), + action: Some(STS_WEB_IDENTITY_ACTION.to_string()), + web_identity_token: Some("sa-token".to_string()), + duration_seconds: None, + policy: None, + }; + let parsed = parse_sts_form("tenant-a".to_string(), "rustfs-a".to_string(), form).unwrap(); + + let response = process_assume_role_request(&runtime, parsed).await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + + #[tokio::test] + async fn sts_handler_merges_binding_policy_and_requests_and_succeeds() { + let runtime = MockStsRuntime { + identity: Mutex::new(Some(Ok(token_review::ServiceAccountIdentity { + namespace: "tenant-a".to_string(), + service_account: "workload-sa".to_string(), + }))), + policy_bindings: Mutex::new(Some(Ok(vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "workload-sa".to_string(), + }, + policies: vec!["policy-binding".to_string()], + }, + status: None, + }]))), + tenant: Mutex::new(Some(Ok(crate::types::v1alpha1::tenant::Tenant { + metadata: Default::default(), + spec: Default::default(), + status: None, + }))), + create_client: Mutex::new(Some(Ok(RustfsAdminClient::new_with_base_url( + "http://127.0.0.1:1", + "access-key", + "secret-key", + )))), + fetch_policy_results: Mutex::new(VecDeque::from([ + Ok("{\"Version\": \"2012-10-17\", \"Statement\": [{\"Sid\":\"b1\",\"Effect\":\"Allow\"}]}".to_string()), + ])), + assume_role_result: Mutex::new(Some(Ok(StsAssumeRoleCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string(), + session_token: "session".to_string(), + expiration: "2024-01-01T00:00:00Z".to_string(), + }))), + }; + + let form = AssumeRoleWithWebIdentityForm { + version: Some(STS_API_VERSION.to_string()), + action: Some(STS_WEB_IDENTITY_ACTION.to_string()), + web_identity_token: Some("sa-token".to_string()), + duration_seconds: Some("3600".to_string()), + policy: Some("{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"req\",\"Effect\":\"Allow\"}] }".to_string()), + }; + let parsed = parse_sts_form("tenant-a".to_string(), "tenant-a".to_string(), form).unwrap(); + + let response = process_assume_role_request(&runtime, parsed).await; + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let text = String::from_utf8(body.to_vec()).unwrap(); + + assert!(text.contains("ak")); + } + + #[tokio::test] + async fn sts_handler_rejects_matching_binding_without_valid_policies() { + let runtime = MockStsRuntime { + identity: Mutex::new(Some(Ok(token_review::ServiceAccountIdentity { + namespace: "tenant-a".to_string(), + service_account: "workload-sa".to_string(), + }))), + policy_bindings: Mutex::new(Some(Ok(vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "workload-sa".to_string(), + }, + policies: vec![], + }, + status: None, + }]))), + tenant: Mutex::new(Some(Ok(crate::types::v1alpha1::tenant::Tenant { + metadata: Default::default(), + spec: Default::default(), + status: None, + }))), + create_client: Mutex::new(Some(Ok(RustfsAdminClient::new_with_base_url( + "http://127.0.0.1:1", + "access-key", + "secret-key", + )))), + fetch_policy_results: Mutex::new(VecDeque::new()), + assume_role_result: Mutex::new(None), + }; + + let form = AssumeRoleWithWebIdentityForm { + version: Some(STS_API_VERSION.to_string()), + action: Some(STS_WEB_IDENTITY_ACTION.to_string()), + web_identity_token: Some("sa-token".to_string()), + duration_seconds: Some("3600".to_string()), + policy: None, + }; + let parsed = parse_sts_form("tenant-a".to_string(), "rustfs-a".to_string(), form).unwrap(); + + let response = process_assume_role_request(&runtime, parsed).await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + + #[tokio::test] + async fn sts_handler_rejects_mtls_tenant_with_clear_bad_request() { + let runtime = MockStsRuntime { + identity: Mutex::new(Some(Ok(token_review::ServiceAccountIdentity { + namespace: "tenant-a".to_string(), + service_account: "workload-sa".to_string(), + }))), + policy_bindings: Mutex::new(Some(Ok(vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "workload-sa".to_string(), + }, + policies: vec!["policy-a".to_string()], + }, + status: None, + }]))), + tenant: Mutex::new(Some(Ok(crate::types::v1alpha1::tenant::Tenant { + metadata: Default::default(), + spec: Default::default(), + status: None, + }))), + create_client: Mutex::new(Some(Err(StsError::TenantTlsClientCertificateUnsupported))), + fetch_policy_results: Mutex::new(VecDeque::new()), + assume_role_result: Mutex::new(None), + }; + + let form = AssumeRoleWithWebIdentityForm { + version: Some(STS_API_VERSION.to_string()), + action: Some(STS_WEB_IDENTITY_ACTION.to_string()), + web_identity_token: Some("sa-token".to_string()), + duration_seconds: Some("3600".to_string()), + policy: None, + }; + let parsed = parse_sts_form("tenant-a".to_string(), "rustfs-a".to_string(), form).unwrap(); + + let response = process_assume_role_request(&runtime, parsed).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let text = String::from_utf8(body.to_vec()).unwrap(); + assert!(text.contains("TenantTlsClientCertificateUnsupported")); + assert!(text.contains("does not support Tenants that require TLS client certificates")); + } + + #[test] + fn mtls_tenant_client_error_maps_to_explicit_sts_error() { + let error = + map_rustfs_client_creation_error(RustfsClientError::TenantTlsClientCertificateRequired); + + assert_eq!(error, StsError::TenantTlsClientCertificateUnsupported); + assert_eq!(sts_error_status(&error), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn resolve_binding_policies_skips_invalid_binding_policies() { + let runtime = MockStsRuntime { + identity: Mutex::new(None), + policy_bindings: Mutex::new(None), + tenant: Mutex::new(None), + create_client: Mutex::new(None), + fetch_policy_results: Mutex::new(VecDeque::from([ + Ok("{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"good\",\"Effect\":\"Allow\"}]}".to_string()), + Ok("{\"Version\":\"2012-10-17\",\"Statement\":[]}".to_string()), + Err(RustfsClientError::RequestFailed), + ])), + assume_role_result: Mutex::new(None), + }; + + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + let bindings = vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "sa".to_string(), + }, + policies: vec![ + "policy-good".to_string(), + "policy-empty".to_string(), + "policy-fail".to_string(), + ], + }, + status: None, + }]; + + let policies = super::resolve_binding_policies(&runtime, &client, &bindings) + .await + .expect("valid referenced policies should allow the STS request"); + + assert_eq!(policies.len(), 1); + assert!(policies[0].contains("\"Sid\":\"good\"")); + } + + #[tokio::test] + async fn resolve_binding_policies_rejects_when_no_valid_binding_policy_exists() { + let runtime = MockStsRuntime { + identity: Mutex::new(None), + policy_bindings: Mutex::new(None), + tenant: Mutex::new(None), + create_client: Mutex::new(None), + fetch_policy_results: Mutex::new(VecDeque::from([Err( + RustfsClientError::RequestFailed, + )])), + assume_role_result: Mutex::new(None), + }; + + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + let bindings = vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "sa".to_string(), + }, + policies: vec!["policy-fail".to_string()], + }, + status: None, + }]; + + let error = super::resolve_binding_policies(&runtime, &client, &bindings) + .await + .expect_err("no valid binding policy must reject the STS request"); + + assert!(matches!(error, StsError::AccessDenied)); + } + + #[tokio::test] + async fn resolve_binding_policies_rejects_empty_binding_policy_lists() { + let runtime = MockStsRuntime { + identity: Mutex::new(None), + policy_bindings: Mutex::new(None), + tenant: Mutex::new(None), + create_client: Mutex::new(None), + fetch_policy_results: Mutex::new(VecDeque::new()), + assume_role_result: Mutex::new(None), + }; + + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + let bindings = vec![PolicyBinding { + metadata: Default::default(), + spec: PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "tenant-a".to_string(), + serviceaccount: "sa".to_string(), + }, + policies: vec![], + }, + status: None, + }]; + + let error = super::resolve_binding_policies(&runtime, &client, &bindings) + .await + .expect_err("empty binding policy list must reject the STS request"); + + assert!(matches!(error, StsError::AccessDenied)); + } + + struct MockStsRuntime { + identity: Mutex>>, + policy_bindings: Mutex, StsError>>>, + tenant: Mutex>>, + create_client: Mutex>>, + fetch_policy_results: Mutex>>, + assume_role_result: + Mutex>>, + } + + #[async_trait] + impl StsRuntime for MockStsRuntime { + async fn authenticate_service_account( + &self, + _token: &str, + _audience: &str, + ) -> Result { + self.identity + .lock() + .unwrap() + .take() + .unwrap_or(Err(StsError::InternalError)) + } + + async fn list_policy_bindings( + &self, + _tenant_namespace: &str, + ) -> Result, StsError> { + self.policy_bindings + .lock() + .unwrap() + .take() + .unwrap_or(Err(StsError::InternalError)) + } + + async fn select_tenant( + &self, + _tenant_namespace: &str, + _tenant_name: &str, + ) -> Result { + self.tenant + .lock() + .unwrap() + .take() + .unwrap_or(Err(StsError::InternalError)) + } + + async fn create_rustfs_admin_client( + &self, + _tenant: &Tenant, + ) -> Result { + self.create_client + .lock() + .unwrap() + .take() + .unwrap_or(Err(StsError::InternalError)) + } + + async fn fetch_canned_policy( + &self, + _rustfs_client: &RustfsAdminClient, + _policy_name: &str, + ) -> Result { + self.fetch_policy_results + .lock() + .unwrap() + .pop_front() + .unwrap_or(Err(RustfsClientError::RequestFailed)) + } + + async fn assume_role( + &self, + _rustfs_client: &RustfsAdminClient, + _policy: Option<&str>, + _duration_seconds: u64, + ) -> Result { + self.assume_role_result + .lock() + .unwrap() + .take() + .unwrap_or(Err(RustfsClientError::RequestFailed)) + } + } +} diff --git a/src/sts/session_policy.rs b/src/sts/session_policy.rs new file mode 100644 index 0000000..371d89d --- /dev/null +++ b/src/sts/session_policy.rs @@ -0,0 +1,201 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde_json::{Map, Value}; + +use crate::sts::error::StsError; + +const DEFAULT_POLICY_VERSION: &str = "2012-10-17"; +pub const MAX_SESSION_POLICY_SIZE: usize = 2048; + +#[derive(Debug, Clone)] +struct ParsedPolicy { + version: String, + statements: Vec, +} + +/// Parse a policy payload and return a normalized compact policy string. +pub fn normalize_policy_for_merge(raw: &str) -> Result { + let policy = parse_policy(raw)?; + + let mut merged = Map::new(); + merged.insert("Version".to_string(), Value::String(policy.version)); + merged.insert("Statement".to_string(), Value::Array(policy.statements)); + + serde_json::to_string(&Value::Object(merged)).map_err(|_| StsError::MalformedPolicyDocument) +} + +/// Merge request policy + binding policies to an inline session policy. +pub fn merge_session_policies( + request_policy: Option<&str>, + binding_policies: &[String], +) -> Result, StsError> { + let mut statements = Vec::::new(); + let mut version: Option = None; + + if let Some(raw_request_policy) = request_policy { + let policy = parse_policy(raw_request_policy)?; + version.get_or_insert(policy.version); + statements.extend(policy.statements); + } + + for raw_policy in binding_policies { + let policy = parse_policy(raw_policy)?; + if version.is_none() { + version = Some(policy.version); + } + statements.extend(policy.statements); + } + + if statements.is_empty() { + return Ok(None); + } + + let mut merged = Map::new(); + merged.insert( + "Version".to_string(), + Value::String(version.unwrap_or_else(|| DEFAULT_POLICY_VERSION.to_string())), + ); + merged.insert("Statement".to_string(), Value::Array(statements)); + + let compacted = serde_json::to_string(&Value::Object(merged)) + .map_err(|_| StsError::MalformedPolicyDocument)?; + + if compacted.len() > MAX_SESSION_POLICY_SIZE { + return Err(StsError::PackedPolicyTooLarge); + } + + Ok(Some(compacted)) +} + +fn parse_policy(raw: &str) -> Result { + let raw = raw.trim(); + if raw.is_empty() { + return Err(StsError::MalformedPolicyDocument); + } + + let raw_policy = + serde_json::from_str::(raw).map_err(|_| StsError::MalformedPolicyDocument)?; + let object = raw_policy + .as_object() + .ok_or(StsError::MalformedPolicyDocument)?; + + let version = object + .get("Version") + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .unwrap_or(DEFAULT_POLICY_VERSION) + .to_string(); + + let statement = object + .get("Statement") + .ok_or(StsError::MalformedPolicyDocument)?; + let statements = match statement { + Value::Array(values) => { + if values.is_empty() { + return Err(StsError::MalformedPolicyDocument); + } + values.clone() + } + Value::Object(object) => vec![Value::Object(object.clone())], + _ => return Err(StsError::MalformedPolicyDocument), + }; + + Ok(ParsedPolicy { + version, + statements, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy(statements: &[&str]) -> String { + let statement_lines = statements + .iter() + .map(|value| format!("{{\"Sid\":\"{value}\",\"Effect\":\"Allow\"}}")) + .collect::>() + .join(","); + + format!("{{\"Version\":\"2012-10-17\",\"Statement\":[{statement_lines}]}}") + } + + #[test] + fn parse_and_compact_policy_rejects_empty_policy() { + assert!(matches!( + normalize_policy_for_merge(""), + Err(StsError::MalformedPolicyDocument) + )); + } + + #[test] + fn parse_and_compact_policy_rejects_malformed_json() { + assert!(matches!( + normalize_policy_for_merge("{\"Version\": \"2012-10-17\""), + Err(StsError::MalformedPolicyDocument) + )); + } + + #[test] + fn parse_and_compact_policy_rejects_missing_statements() { + let without_statements = "{\"Version\":\"2012-10-17\"}"; + assert!(matches!( + normalize_policy_for_merge(without_statements), + Err(StsError::MalformedPolicyDocument) + )); + } + + #[test] + fn merge_request_and_binding_policies_keeps_compact_shape() { + let request_policy = policy(&["RequestPolicy"]); + let binding_policy = policy(&["BindingPolicy"]); + + let merged = merge_session_policies(Some(&request_policy), &[binding_policy]) + .expect("merge should succeed"); + let merged = merged.expect("merged policy should exist"); + + let value = serde_json::from_str::(&merged).expect("merged policy is json"); + assert_eq!(value["Version"], Value::String("2012-10-17".to_string())); + let statements = value["Statement"] + .as_array() + .expect("merged policy should contain statement array"); + assert_eq!(statements.len(), 2); + assert!(merged.len() <= MAX_SESSION_POLICY_SIZE); + } + + #[test] + fn merge_session_policy_returns_none_for_empty_inputs() { + let merged = merge_session_policies(None, &[]).expect("merge should succeed"); + assert!(merged.is_none()); + } + + #[test] + fn merge_rejects_oversized_inline_policy() { + let long_statement = policy(&["A"; 4000]); + + let err = merge_session_policies(Some(&long_statement), &[]) + .expect_err("policy should be too large"); + assert!(matches!(err, StsError::PackedPolicyTooLarge)); + } + + #[test] + fn merge_skips_no_statement_policy_as_malformed() { + let no_statements = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + assert!(matches!( + normalize_policy_for_merge(no_statements), + Err(StsError::MalformedPolicyDocument) + )); + } +} diff --git a/src/sts/tls.rs b/src/sts/tls.rs new file mode 100644 index 0000000..18f1e70 --- /dev/null +++ b/src/sts/tls.rs @@ -0,0 +1,486 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::io::Cursor; +use std::net::Ipv4Addr; +use std::time::Duration; + +use k8s_openapi::ByteString; +use k8s_openapi::api::core::v1 as corev1; +use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; +use kube::api::PostParams; +use kube::{Api, Client}; +use rcgen::{ + BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose, +}; +use rustls::pki_types::CertificateDer; +use snafu::{OptionExt, ResultExt, Snafu}; +use tokio::time::sleep; +use tracing::{info, warn}; + +const STS_TLS_SECRET_NAME: &str = "sts-tls"; +const DEFAULT_STS_SERVICE_NAME: &str = "rustfs-operator-sts"; +const DEFAULT_OPERATOR_NAMESPACE: &str = "rustfs-system"; +const SERVICE_ACCOUNT_NAMESPACE_PATH: &str = + "/var/run/secrets/kubernetes.io/serviceaccount/namespace"; +const TLS_CERT_KEY: &str = "tls.crt"; +const TLS_KEY_KEY: &str = "tls.key"; +const CA_CERT_KEY: &str = "ca.crt"; +const MANAGED_LABEL: &str = "operator.rustfs.com/managed-sts-tls"; +const KUBERNETES_TLS_SECRET_TYPE: &str = "kubernetes.io/tls"; +const SECRET_WAIT_ATTEMPTS: usize = 30; +const SECRET_WAIT_INTERVAL: Duration = Duration::from_secs(2); + +pub type TlsResult = Result; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + #[snafu(display( + "operator STS TLS Secret {namespace}/{secret} was not found and OPERATOR_STS_TLS_AUTO=false" + ))] + SecretNotFound { namespace: String, secret: String }, + + #[snafu(display("timed out waiting for operator STS TLS Secret {namespace}/{secret}"))] + SecretWaitTimedOut { namespace: String, secret: String }, + + #[snafu(display("failed to {action} operator STS TLS Secret {namespace}/{secret}: {source}"))] + Kube { + source: Box, + action: &'static str, + namespace: String, + secret: String, + }, + + #[snafu(display("operator STS TLS Secret {namespace}/{secret} has no data"))] + SecretNoData { namespace: String, secret: String }, + + #[snafu(display("operator STS TLS Secret {namespace}/{secret} is missing non-empty {key}"))] + SecretMissingKey { + namespace: String, + secret: String, + key: &'static str, + }, + + #[snafu(display("operator STS TLS Secret {namespace}/{secret} is missing ca.crt or tls.crt"))] + SecretMissingCa { namespace: String, secret: String }, + + #[snafu(display("failed to generate operator STS TLS certificate: {source}"))] + GenerateCertificate { source: rcgen::Error }, + + #[snafu(display("failed to parse STS TLS certificate: {source}"))] + ParseCertificate { source: std::io::Error }, + + #[snafu(display("STS TLS certificate bundle is empty"))] + EmptyCertificateBundle, + + #[snafu(display("failed to parse STS TLS private key: {source}"))] + ParsePrivateKey { source: std::io::Error }, + + #[snafu(display("STS TLS private key is missing"))] + MissingPrivateKey, + + #[snafu(display("failed to build STS TLS server config: {source}"))] + BuildServerConfig { source: rustls::Error }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OperatorStsTlsConfig { + pub enabled: bool, + pub auto_generate: bool, + pub namespace: String, + pub service_name: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OperatorStsTlsMaterial { + pub secret_name: String, + pub cert_pem: Vec, + pub key_pem: Vec, + pub ca_pem: Vec, +} + +impl OperatorStsTlsConfig { + pub fn from_env() -> Self { + Self { + enabled: env_bool("OPERATOR_STS_TLS_ENABLED", true), + auto_generate: env_bool("OPERATOR_STS_TLS_AUTO", true), + namespace: operator_namespace(), + service_name: env_string("OPERATOR_STS_SERVICE_NAME", DEFAULT_STS_SERVICE_NAME), + } + } +} + +pub async fn load_or_create_sts_tls_material( + client: &Client, + config: &OperatorStsTlsConfig, +) -> TlsResult { + let api: Api = Api::namespaced(client.clone(), &config.namespace); + + match api.get(STS_TLS_SECRET_NAME).await { + Ok(secret) => load_material_from_secret_or_regenerate(&api, config, secret).await, + Err(kube::Error::Api(error)) if error.code == 404 && config.auto_generate => { + create_or_get_generated_secret(&api, config).await + } + Err(kube::Error::Api(error)) if error.code == 404 => SecretNotFoundSnafu { + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + } + .fail(), + Err(source) => Err(Error::Kube { + source: Box::new(source), + action: "load", + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + }), + } +} + +pub fn build_tls_server_config( + material: &OperatorStsTlsMaterial, +) -> TlsResult { + crate::install_rustls_crypto_provider(); + + let certs = rustls_pemfile::certs(&mut Cursor::new(&material.cert_pem)) + .collect::>, _>>() + .context(ParseCertificateSnafu)?; + if certs.is_empty() { + return EmptyCertificateBundleSnafu.fail(); + } + + let key = rustls_pemfile::private_key(&mut Cursor::new(&material.key_pem)) + .context(ParsePrivateKeySnafu)? + .context(MissingPrivateKeySnafu)?; + + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .context(BuildServerConfigSnafu) +} + +async fn load_material_from_secret_or_regenerate( + api: &Api, + config: &OperatorStsTlsConfig, + secret: corev1::Secret, +) -> TlsResult { + match material_from_secret(config, &secret) { + Ok(material) => Ok(material), + Err(error) if config.auto_generate && is_operator_managed(&secret) => { + warn!( + secret = STS_TLS_SECRET_NAME, + %error, + "regenerating invalid managed operator STS TLS Secret" + ); + replace_generated_secret(api, config, &secret).await + } + Err(error) => Err(error), + } +} + +async fn create_or_get_generated_secret( + api: &Api, + config: &OperatorStsTlsConfig, +) -> TlsResult { + let generated = generated_sts_tls_secret(config)?; + match api.create(&PostParams::default(), &generated).await { + Ok(secret) => { + info!( + secret = STS_TLS_SECRET_NAME, + namespace = %config.namespace, + "created operator STS TLS Secret" + ); + material_from_secret(config, &secret) + } + Err(kube::Error::Api(error)) if error.code == 409 => { + wait_for_secret_material(api, config).await + } + Err(source) => Err(Error::Kube { + source: Box::new(source), + action: "create", + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + }), + } +} + +async fn replace_generated_secret( + api: &Api, + config: &OperatorStsTlsConfig, + existing: &corev1::Secret, +) -> TlsResult { + let mut generated = generated_sts_tls_secret(config)?; + generated.metadata.resource_version = existing.metadata.resource_version.clone(); + match api + .replace(STS_TLS_SECRET_NAME, &PostParams::default(), &generated) + .await + { + Ok(secret) => material_from_secret(config, &secret), + Err(source) => Err(Error::Kube { + source: Box::new(source), + action: "replace managed", + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + }), + } +} + +async fn wait_for_secret_material( + api: &Api, + config: &OperatorStsTlsConfig, +) -> TlsResult { + for _ in 0..SECRET_WAIT_ATTEMPTS { + match api.get(STS_TLS_SECRET_NAME).await { + Ok(secret) => return material_from_secret(config, &secret), + Err(kube::Error::Api(error)) if error.code == 404 => { + sleep(SECRET_WAIT_INTERVAL).await; + } + Err(source) => { + return Err(Error::Kube { + source: Box::new(source), + action: "wait for", + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + }); + } + } + } + + SecretWaitTimedOutSnafu { + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + } + .fail() +} + +fn generated_sts_tls_secret(config: &OperatorStsTlsConfig) -> TlsResult { + let generated = generate_sts_tls_material(&config.namespace, &config.service_name)?; + let mut data = BTreeMap::new(); + data.insert(TLS_CERT_KEY.to_string(), ByteString(generated.cert_pem)); + data.insert(TLS_KEY_KEY.to_string(), ByteString(generated.key_pem)); + data.insert(CA_CERT_KEY.to_string(), ByteString(generated.ca_pem)); + + let mut labels = BTreeMap::new(); + labels.insert(MANAGED_LABEL.to_string(), "true".to_string()); + labels.insert( + "app.kubernetes.io/name".to_string(), + "rustfs-operator".to_string(), + ); + labels.insert( + "app.kubernetes.io/component".to_string(), + "operator".to_string(), + ); + + Ok(corev1::Secret { + metadata: metav1::ObjectMeta { + name: Some(STS_TLS_SECRET_NAME.to_string()), + namespace: Some(config.namespace.clone()), + labels: Some(labels), + ..Default::default() + }, + type_: Some(KUBERNETES_TLS_SECRET_TYPE.to_string()), + data: Some(data), + ..Default::default() + }) +} + +fn generate_sts_tls_material( + namespace: &str, + service_name: &str, +) -> TlsResult { + let ca_key = KeyPair::generate().context(GenerateCertificateSnafu)?; + let mut ca_params = CertificateParams::default(); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + ca_params.key_usages = vec![ + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::CrlSign, + ]; + let ca_cert = ca_params + .self_signed(&ca_key) + .context(GenerateCertificateSnafu)?; + + let server_key = KeyPair::generate().context(GenerateCertificateSnafu)?; + let mut server_names = service_dns_names(namespace, service_name); + server_names.push("localhost".to_string()); + server_names.push(Ipv4Addr::LOCALHOST.to_string()); + let mut server_params = + CertificateParams::new(server_names).context(GenerateCertificateSnafu)?; + server_params.is_ca = IsCa::NoCa; + server_params.key_usages = vec![ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyEncipherment, + ]; + server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let server_cert = server_params + .signed_by(&server_key, &ca_cert, &ca_key) + .context(GenerateCertificateSnafu)?; + + Ok(OperatorStsTlsMaterial { + secret_name: STS_TLS_SECRET_NAME.to_string(), + cert_pem: server_cert.pem().into_bytes(), + key_pem: server_key.serialize_pem().into_bytes(), + ca_pem: ca_cert.pem().into_bytes(), + }) +} + +fn material_from_secret( + config: &OperatorStsTlsConfig, + secret: &corev1::Secret, +) -> TlsResult { + let data = secret.data.as_ref().context(SecretNoDataSnafu { + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + })?; + + let cert_pem = secret_data(data, TLS_CERT_KEY, config)?; + let key_pem = secret_data(data, TLS_KEY_KEY, config)?; + let ca_pem = data + .get(CA_CERT_KEY) + .or_else(|| data.get(TLS_CERT_KEY)) + .map(|bytes| bytes.0.clone()) + .context(SecretMissingCaSnafu { + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + })?; + + Ok(OperatorStsTlsMaterial { + secret_name: STS_TLS_SECRET_NAME.to_string(), + cert_pem, + key_pem, + ca_pem, + }) +} + +fn secret_data( + data: &BTreeMap, + key: &'static str, + config: &OperatorStsTlsConfig, +) -> TlsResult> { + data.get(key) + .map(|bytes| bytes.0.clone()) + .filter(|bytes| !bytes.is_empty()) + .context(SecretMissingKeySnafu { + namespace: config.namespace.clone(), + secret: STS_TLS_SECRET_NAME.to_string(), + key, + }) +} + +fn service_dns_names(namespace: &str, service_name: &str) -> Vec { + vec![ + service_name.to_string(), + format!("{service_name}.{namespace}"), + format!("{service_name}.{namespace}.svc"), + format!("{service_name}.{namespace}.svc.cluster.local"), + ] +} + +fn is_operator_managed(secret: &corev1::Secret) -> bool { + secret + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(MANAGED_LABEL)) + .is_some_and(|value| value == "true") +} + +fn operator_namespace() -> String { + if let Some(value) = std::env::var("OPERATOR_NAMESPACE") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + { + return value; + } + + std::fs::read_to_string(SERVICE_ACCOUNT_NAMESPACE_PATH) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_OPERATOR_NAMESPACE.to_string()) +} + +fn env_bool(name: &str, default: bool) -> bool { + std::env::var(name) + .ok() + .and_then(|value| match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Some(true), + "0" | "false" | "no" | "off" => Some(false), + _ => None, + }) + .unwrap_or(default) +} + +fn env_string(name: &str, default: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn service_dns_names_cover_short_and_cluster_forms() { + assert_eq!( + service_dns_names("rustfs-system", "rustfs-operator-sts"), + vec![ + "rustfs-operator-sts", + "rustfs-operator-sts.rustfs-system", + "rustfs-operator-sts.rustfs-system.svc", + "rustfs-operator-sts.rustfs-system.svc.cluster.local" + ] + ); + } + + #[test] + fn generated_material_builds_rustls_server_config() { + let material = generate_sts_tls_material("rustfs-system", "rustfs-operator-sts").unwrap(); + + assert!(!material.cert_pem.is_empty()); + assert!(!material.key_pem.is_empty()); + assert!(!material.ca_pem.is_empty()); + build_tls_server_config(&material).unwrap(); + } + + #[test] + fn secret_material_uses_leaf_as_ca_fallback() { + let config = OperatorStsTlsConfig { + enabled: true, + auto_generate: true, + namespace: "rustfs-system".to_string(), + service_name: "rustfs-operator-sts".to_string(), + }; + let generated = generate_sts_tls_material(&config.namespace, &config.service_name).unwrap(); + let mut data = BTreeMap::new(); + data.insert(TLS_CERT_KEY.to_string(), ByteString(generated.cert_pem)); + data.insert(TLS_KEY_KEY.to_string(), ByteString(generated.key_pem)); + let secret = corev1::Secret { + metadata: metav1::ObjectMeta { + name: Some(STS_TLS_SECRET_NAME.to_string()), + namespace: Some(config.namespace.clone()), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + + let material = material_from_secret(&config, &secret).unwrap(); + assert_eq!(material.ca_pem, material.cert_pem); + } +} diff --git a/src/sts/token_review.rs b/src/sts/token_review.rs new file mode 100644 index 0000000..f6a01da --- /dev/null +++ b/src/sts/token_review.rs @@ -0,0 +1,235 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use k8s_openapi::api::authentication::v1::{TokenReviewStatus, UserInfo}; + +/// Prefix used by Kubernetes for service account usernames. +const SERVICE_ACCOUNT_USERNAME_PREFIX: &str = "system:serviceaccount:"; + +/// Service account identity extracted from a TokenReview response. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServiceAccountIdentity { + pub namespace: String, + pub service_account: String, +} + +/// Errors that can happen while processing Kubernetes TokenReview. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TokenReviewError { + MissingTokenReview, + NotAuthenticated, + MissingAudience, + InvalidAudience, + MissingUsername, + InvalidUsername, + InvalidUsernameFormat, +} + +impl TokenReviewError { + pub fn as_message(&self) -> String { + match self { + Self::MissingTokenReview => "TokenReview response is incomplete.".to_string(), + Self::NotAuthenticated => "TokenReview failed to authenticate token.".to_string(), + Self::MissingAudience => "TokenReview response has no accepted audience.".to_string(), + Self::InvalidAudience => { + "TokenReview response did not accept the expected audience.".to_string() + } + Self::MissingUsername => "TokenReview response has no user identity.".to_string(), + Self::InvalidUsername => { + "TokenReview returned an invalid service account user name.".to_string() + } + Self::InvalidUsernameFormat => { + "TokenReview service account user name format is invalid.".to_string() + } + } + } +} + +/// Parse `system:serviceaccount::` into namespace/SA fields. +pub fn parse_service_account_username( + raw_username: &str, +) -> Result { + let raw = raw_username.trim(); + let remaining = raw + .strip_prefix(SERVICE_ACCOUNT_USERNAME_PREFIX) + .ok_or(TokenReviewError::InvalidUsername)?; + + let mut parts = remaining.split(':'); + let namespace = parts + .next() + .ok_or(TokenReviewError::InvalidUsernameFormat)?; + let service_account = parts + .next() + .ok_or(TokenReviewError::InvalidUsernameFormat)?; + if parts.next().is_some() { + return Err(TokenReviewError::InvalidUsernameFormat); + } + + if namespace.is_empty() || service_account.is_empty() { + return Err(TokenReviewError::InvalidUsernameFormat); + } + + Ok(ServiceAccountIdentity { + namespace: namespace.to_string(), + service_account: service_account.to_string(), + }) +} + +/// Convert TokenReview status payload to service account identity. +pub fn extract_service_account_identity( + status: &TokenReviewStatus, +) -> Result { + extract_service_account_identity_for_audience(status, None) +} + +/// Convert TokenReview status payload to service account identity and validate accepted audience. +pub fn extract_service_account_identity_for_audience( + status: &TokenReviewStatus, + expected_audience: Option<&str>, +) -> Result { + if !status.authenticated.unwrap_or(false) { + return Err(TokenReviewError::NotAuthenticated); + } + + if let Some(expected_audience) = expected_audience { + let audiences = status + .audiences + .as_ref() + .ok_or(TokenReviewError::MissingAudience)?; + if !audiences + .iter() + .any(|audience| audience == expected_audience) + { + return Err(TokenReviewError::InvalidAudience); + } + } + + let user = status + .user + .as_ref() + .ok_or(TokenReviewError::MissingUsername)?; + let username = user + .username + .as_ref() + .ok_or(TokenReviewError::MissingUsername)?; + parse_service_account_username(username) +} + +/// Build a synthetic TokenReviewStatus for unit tests. +/// +/// The parser helpers above are intentionally transport-agnostic and can be reused by both +/// mock and live flows. +pub fn token_review_status(authenticated: bool, username: Option<&str>) -> TokenReviewStatus { + TokenReviewStatus { + audiences: None, + authenticated: Some(authenticated), + error: None, + user: username.map(|value| UserInfo { + extra: None, + groups: None, + uid: None, + username: Some(value.to_string()), + }), + } +} + +#[cfg(test)] +mod tests { + use super::{TokenReviewError, parse_service_account_username, token_review_status}; + use k8s_openapi::api::authentication::v1::TokenReviewStatus; + + #[test] + fn parse_service_account_username_success() { + let identity = parse_service_account_username("system:serviceaccount:tenant-ns:tenant-sa") + .expect("service account identity should parse"); + + assert_eq!(identity.namespace, "tenant-ns"); + assert_eq!(identity.service_account, "tenant-sa"); + } + + #[test] + fn parse_service_account_username_requires_service_account_prefix() { + let error = parse_service_account_username("kube-system:tenant-sa") + .expect_err("identity should require serviceaccount prefix"); + + assert!(matches!(error, TokenReviewError::InvalidUsername)); + } + + #[test] + fn parse_service_account_username_rejects_non_two_parts() { + let error = parse_service_account_username("system:serviceaccount:tenant-sa") + .expect_err("identity should require namespace and serviceaccount"); + + assert!(matches!(error, TokenReviewError::InvalidUsernameFormat)); + } + + #[test] + fn extract_service_account_identity_rejects_unauthenticated_token() { + let status = token_review_status(false, Some("system:serviceaccount:tenant-ns:tenant-sa")); + + let err = super::extract_service_account_identity(&status) + .expect_err("unauthenticated token should fail"); + + assert!(matches!(err, TokenReviewError::NotAuthenticated)); + } + + #[test] + fn extract_service_account_identity_rejects_non_service_account_usernames() { + let status = token_review_status(true, Some("invalid-user")); + + let err = super::extract_service_account_identity(&status) + .expect_err("only system:serviceaccount users are supported"); + + assert!(matches!(err, TokenReviewError::InvalidUsername)); + } + + #[test] + fn extract_service_account_identity_rejects_missing_user_field() { + let status = token_review_status(true, None); + + let err = super::extract_service_account_identity(&status) + .expect_err("missing user payload should fail"); + + assert!(matches!(err, TokenReviewError::MissingUsername)); + } + + #[test] + fn extract_service_account_identity_requires_expected_audience() { + let status = TokenReviewStatus { + audiences: Some(vec!["sts.rustfs.com".to_string()]), + ..token_review_status(true, Some("system:serviceaccount:tenant-ns:tenant-sa")) + }; + + let identity = + super::extract_service_account_identity_for_audience(&status, Some("sts.rustfs.com")) + .expect("matching audience should be accepted"); + + assert_eq!(identity.namespace, "tenant-ns"); + assert_eq!(identity.service_account, "tenant-sa"); + } + + #[test] + fn extract_service_account_identity_rejects_wrong_audience() { + let status = TokenReviewStatus { + audiences: Some(vec!["https://kubernetes.default.svc".to_string()]), + ..token_review_status(true, Some("system:serviceaccount:tenant-ns:tenant-sa")) + }; + + let err = + super::extract_service_account_identity_for_audience(&status, Some("sts.rustfs.com")) + .expect_err("wrong audience should fail"); + + assert!(matches!(err, TokenReviewError::InvalidAudience)); + } +} diff --git a/src/sts/types.rs b/src/sts/types.rs new file mode 100644 index 0000000..1f2c224 --- /dev/null +++ b/src/sts/types.rs @@ -0,0 +1,452 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::Deserialize; + +use crate::sts::error::{STS_REQUEST_ID, STS_XML_NAMESPACE, StsError, escape_xml}; + +pub const STS_API_VERSION: &str = "2011-06-15"; +pub const STS_WEB_IDENTITY_ACTION: &str = "AssumeRoleWithWebIdentity"; +pub const STS_DEFAULT_DURATION_SECONDS: u64 = 3600; +pub const STS_MIN_DURATION_SECONDS: u64 = 900; +pub const STS_MAX_DURATION_SECONDS: u64 = 31_536_000; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StsParsedRequest { + pub tenant_namespace: String, + pub tenant_name: String, + pub version: String, + pub action: String, + pub web_identity_token: String, + pub duration_seconds: u64, + pub policy: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct AssumeRoleWithWebIdentityForm { + pub version: Option, + pub action: Option, + pub web_identity_token: Option, + pub duration_seconds: Option, + pub policy: Option, +} + +/// Parse a form-style STS request and validate required parameters. +pub fn parse_sts_form( + tenant_namespace: String, + tenant_name: String, + form: AssumeRoleWithWebIdentityForm, +) -> Result { + let version = form + .version + .filter(|value| !value.trim().is_empty()) + .ok_or(StsError::MissingParameter { + parameter: "Version", + })?; + + if version != STS_API_VERSION { + return Err(StsError::InvalidParameterValue { + parameter: "Version", + }); + } + + let action = + form.action + .filter(|value| !value.trim().is_empty()) + .ok_or(StsError::MissingParameter { + parameter: "Action", + })?; + + if action != STS_WEB_IDENTITY_ACTION { + return Err(StsError::InvalidParameterValue { + parameter: "Action", + }); + } + + let web_identity_token = form + .web_identity_token + .filter(|value| !value.trim().is_empty()) + .ok_or(StsError::MissingParameter { + parameter: "WebIdentityToken", + })?; + + let duration_seconds = match form.duration_seconds { + Some(raw) => { + let duration = + raw.trim() + .parse::() + .map_err(|_| StsError::InvalidParameterValue { + parameter: "DurationSeconds", + })?; + + if !(STS_MIN_DURATION_SECONDS..=STS_MAX_DURATION_SECONDS).contains(&duration) { + return Err(StsError::InvalidParameterValue { + parameter: "DurationSeconds", + }); + } + + duration + } + None => STS_DEFAULT_DURATION_SECONDS, + }; + + Ok(StsParsedRequest { + tenant_namespace, + tenant_name, + version, + action, + web_identity_token, + duration_seconds, + policy: form.policy, + }) +} + +#[derive(Debug, Clone)] +pub struct StsAssumeRoleCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expiration: String, +} + +#[derive(Debug, Clone)] +pub struct StsWebIdentityResponseContext { + pub subject: String, + pub audience: String, + pub provider: String, + pub assumed_role_arn: String, + pub assumed_role_id: String, + pub packed_policy_size: u8, +} + +/// Build an AWS STS AssumeRoleWithWebIdentity success XML payload. +pub fn render_assume_role_with_web_identity_response( + credentials: &StsAssumeRoleCredentials, + context: &StsWebIdentityResponseContext, +) -> String { + let access_key_id = escape_xml(&credentials.access_key_id); + let secret_access_key = escape_xml(&credentials.secret_access_key); + let session_token = escape_xml(&credentials.session_token); + let expiration = escape_xml(&credentials.expiration); + let subject = escape_xml(&context.subject); + let audience = escape_xml(&context.audience); + let provider = escape_xml(&context.provider); + let assumed_role_arn = escape_xml(&context.assumed_role_arn); + let assumed_role_id = escape_xml(&context.assumed_role_id); + let packed_policy_size = context.packed_policy_size; + + format!( + "{subject}{audience}{assumed_role_arn}{assumed_role_id}{access_key_id}{secret_access_key}{session_token}{expiration}{packed_policy_size}{provider}{request_id}", + ns = STS_XML_NAMESPACE, + request_id = STS_REQUEST_ID, + ) +} + +/// Build a deterministic stub XML error for the current implementation phase. +pub fn render_not_implemented_response() -> String { + format!( + "SenderNotImplementedAssumeRoleWithWebIdentity is not implemented in this phase{request_id}", + ns = STS_XML_NAMESPACE, + request_id = STS_REQUEST_ID + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::sts::error::{StsError, escape_xml}; + + fn form( + version: &str, + action: &str, + web_identity_token: &str, + duration: Option<&str>, + policy: Option<&str>, + ) -> AssumeRoleWithWebIdentityForm { + AssumeRoleWithWebIdentityForm { + version: if version.is_empty() { + None + } else { + Some(version.to_string()) + }, + action: if action.is_empty() { + None + } else { + Some(action.to_string()) + }, + web_identity_token: if web_identity_token.is_empty() { + None + } else { + Some(web_identity_token.to_string()) + }, + duration_seconds: duration.map(ToString::to_string), + policy: policy.map(ToString::to_string), + } + } + + #[test] + fn parse_rejects_missing_version() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form("", STS_WEB_IDENTITY_ACTION, "token", None, None), + ); + + assert!(matches!( + request, + Err(StsError::MissingParameter { + parameter: "Version" + }) + )); + } + + #[test] + fn parse_rejects_invalid_version() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form("2010-01-01", STS_WEB_IDENTITY_ACTION, "token", None, None), + ); + + assert!(matches!( + request, + Err(StsError::InvalidParameterValue { + parameter: "Version" + }) + )); + } + + #[test] + fn parse_rejects_missing_action() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form(STS_API_VERSION, "", "token", None, None), + ); + + assert!(matches!( + request, + Err(StsError::MissingParameter { + parameter: "Action" + }) + )); + } + + #[test] + fn parse_rejects_wrong_action() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form(STS_API_VERSION, "ListBuckets", "token", None, None), + ); + + assert!(matches!( + request, + Err(StsError::InvalidParameterValue { + parameter: "Action" + }) + )); + } + + #[test] + fn parse_rejects_missing_web_identity_token() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form(STS_API_VERSION, STS_WEB_IDENTITY_ACTION, "", None, None), + ); + + assert!(matches!( + request, + Err(StsError::MissingParameter { + parameter: "WebIdentityToken" + }) + )); + } + + #[test] + fn parse_rejects_non_integer_duration() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form( + STS_API_VERSION, + STS_WEB_IDENTITY_ACTION, + "token", + Some("not-a-number"), + None, + ), + ); + + assert!(matches!( + request, + Err(StsError::InvalidParameterValue { + parameter: "DurationSeconds" + }) + )); + } + + #[test] + fn parse_rejects_duration_below_minimum() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form( + STS_API_VERSION, + STS_WEB_IDENTITY_ACTION, + "token", + Some("899"), + None, + ), + ); + + assert!(matches!( + request, + Err(StsError::InvalidParameterValue { + parameter: "DurationSeconds" + }) + )); + } + + #[test] + fn parse_rejects_duration_above_maximum() { + let request = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form( + STS_API_VERSION, + STS_WEB_IDENTITY_ACTION, + "token", + Some("31536001"), + None, + ), + ); + + assert!(matches!( + request, + Err(StsError::InvalidParameterValue { + parameter: "DurationSeconds" + }) + )); + } + + #[test] + fn parse_accepts_minimum_and_maximum_duration_bounds() { + let minimum = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form( + STS_API_VERSION, + STS_WEB_IDENTITY_ACTION, + "token", + Some("900"), + None, + ), + ) + .expect("minimum duration is valid"); + let maximum = parse_sts_form( + "tenant-a".to_string(), + "rustfs-a".to_string(), + form( + STS_API_VERSION, + STS_WEB_IDENTITY_ACTION, + "token", + Some("31536000"), + None, + ), + ) + .expect("maximum duration is valid"); + + assert_eq!(minimum.duration_seconds, STS_MIN_DURATION_SECONDS); + assert_eq!(maximum.duration_seconds, STS_MAX_DURATION_SECONDS); + } + + #[test] + fn parse_defaults_duration_to_3600_and_preserves_policy() { + let policy = r#"{\"Statement\": [{\"Action\": \"s3:GetObject\", \"Effect\": \"Allow\"}]}"#; + let request = parse_sts_form( + "tenant-b".to_string(), + "tenant-b".to_string(), + form( + STS_API_VERSION, + STS_WEB_IDENTITY_ACTION, + "token-value", + None, + Some(policy), + ), + ) + .expect("valid request"); + + assert_eq!(request.tenant_namespace, "tenant-b"); + assert_eq!(request.tenant_name, "tenant-b"); + assert_eq!(request.duration_seconds, STS_DEFAULT_DURATION_SECONDS); + assert_eq!(request.policy.as_deref(), Some(policy)); + } + + #[test] + fn error_xml_escapes_and_omit_payload_values() { + let raw = " & token"; + let escaped = escape_xml(raw); + assert_eq!(escaped, "<x> & token"); + + let err = StsError::InvalidParameterValue { + parameter: "DurationSeconds", + } + .as_xml(); + + assert!(err.contains("InvalidParameterValue")); + assert!(!err.contains(" & token")); + } + + #[test] + fn render_success_xml_matches_aws_shape() { + let credentials = StsAssumeRoleCredentials { + access_key_id: "[REDACTED]".to_string(), + secret_access_key: "[REDACTED]".to_string(), + session_token: "[REDACTED]".to_string(), + expiration: "2026-06-20T00:00:00Z".to_string(), + }; + let context = StsWebIdentityResponseContext { + subject: "system:serviceaccount:apps:workload".to_string(), + audience: "sts.rustfs.com".to_string(), + provider: "kubernetes".to_string(), + assumed_role_arn: "arn:rustfs:sts::tenant-a:assumed-role/tenant-a/workload".to_string(), + assumed_role_id: "[REDACTED]:workload".to_string(), + packed_policy_size: 0, + }; + + let xml = render_assume_role_with_web_identity_response(&credentials, &context); + + assert!(xml.contains("")); + assert!(xml.contains("system:serviceaccount:apps:workload")); + assert!(xml.contains("sts.rustfs.com")); + assert!(xml.contains("")); + assert!(xml.contains("")); + assert!(xml.contains("[REDACTED]")); + assert!(xml.contains("0")); + assert!(xml.contains("")); + } + + #[test] + fn render_not_implemented_xml_shape_is_stable() { + let xml = render_not_implemented_response(); + + assert!(xml.contains("NotImplemented")); + assert!(xml.contains(&format!("{}", STS_REQUEST_ID))); + } +} diff --git a/src/types/v1alpha1.rs b/src/types/v1alpha1.rs index 13f73f2..45b3f5f 100755 --- a/src/types/v1alpha1.rs +++ b/src/types/v1alpha1.rs @@ -16,6 +16,7 @@ pub mod encryption; pub mod k8s; pub mod logging; pub mod persistence; +pub mod policy_binding; pub mod pool; pub mod status; pub mod tenant; @@ -23,3 +24,111 @@ pub mod tls; // Re-export commonly used types pub use pool::{SchedulingConfig, validate_pool_total_volumes}; + +#[cfg(test)] +mod policy_binding_tests { + use super::policy_binding::{ + PolicyBinding, PolicyBindingApplication, PolicyBindingSpec, PolicyBindingStatus, + PolicyBindingUsage, + }; + use kube::{CustomResourceExt, Resource}; + use serde_json::json; + + #[test] + fn policy_binding_serializes_minio_aligned_field_names() { + let binding = PolicyBinding::new( + "readonly-binding", + PolicyBindingSpec { + application: PolicyBindingApplication { + namespace: "apps".to_string(), + serviceaccount: "readonly-sa".to_string(), + }, + policies: vec!["readonly".to_string(), "diagnostics".to_string()], + }, + ); + + let value = serde_json::to_value(&binding).expect("PolicyBinding serializes to JSON"); + + assert_eq!(value["apiVersion"], json!("sts.rustfs.com/v1alpha1")); + assert_eq!(value["kind"], json!("PolicyBinding")); + assert_eq!(value["spec"]["application"]["namespace"], json!("apps")); + assert_eq!( + value["spec"]["application"]["serviceaccount"], + json!("readonly-sa") + ); + assert_eq!( + value["spec"]["policies"], + json!(["readonly", "diagnostics"]) + ); + assert!(value["spec"]["application"]["serviceAccount"].is_null()); + } + + #[test] + fn policy_binding_status_serializes_optional_usage_authorizations() { + let status = PolicyBindingStatus { + current_state: Some("Ready".to_string()), + usage: Some(PolicyBindingUsage { + authorizations: Some(3), + }), + }; + + let value = serde_json::to_value(status).expect("PolicyBindingStatus serializes to JSON"); + + assert_eq!(value["currentState"], json!("Ready")); + assert_eq!(value["usage"]["authorizations"], json!(3)); + } + + #[test] + fn policy_binding_crd_has_sts_group_kind_namespaced_scope_and_required_schema() { + let crd = serde_json::to_value(PolicyBinding::crd()).expect("PolicyBinding CRD serializes"); + + assert_eq!(crd["spec"]["group"], json!("sts.rustfs.com")); + assert_eq!(crd["spec"]["names"]["kind"], json!("PolicyBinding")); + assert_eq!(crd["spec"]["scope"], json!("Namespaced")); + + let schema = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]; + assert_eq!(schema["required"], json!(["application", "policies"])); + assert_eq!( + schema["properties"]["application"]["required"], + json!(["namespace", "serviceaccount"]) + ); + assert_eq!( + schema["properties"]["application"]["properties"]["namespace"]["type"], + json!("string") + ); + assert_eq!( + schema["properties"]["application"]["properties"]["serviceaccount"]["type"], + json!("string") + ); + assert_eq!( + schema["properties"]["policies"]["items"]["type"], + json!("string") + ); + assert_eq!( + schema["properties"]["policies"]["x-kubernetes-validations"][0]["rule"], + json!("self.size() > 0") + ); + assert_eq!( + schema["properties"]["policies"]["x-kubernetes-validations"][0]["message"], + json!("policies must contain at least one policy") + ); + + let status_schema = + &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["status"]; + assert_eq!( + status_schema["properties"]["currentState"]["type"], + json!("string") + ); + assert_eq!( + status_schema["properties"]["usage"]["properties"]["authorizations"]["type"], + json!("integer") + ); + } + + #[test] + fn policy_binding_resource_metadata_is_namespaced() { + assert_eq!(PolicyBinding::api_version(&()), "sts.rustfs.com/v1alpha1"); + assert_eq!(PolicyBinding::kind(&()), "PolicyBinding"); + assert_eq!(PolicyBinding::plural(&()), "policybindings"); + } +} diff --git a/src/types/v1alpha1/policy_binding.rs b/src/types/v1alpha1/policy_binding.rs new file mode 100644 index 0000000..077c117 --- /dev/null +++ b/src/types/v1alpha1/policy_binding.rs @@ -0,0 +1,61 @@ +// Copyright 2025 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use kube::{CustomResource, KubeSchema}; +use serde::{Deserialize, Serialize}; + +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, KubeSchema)] +#[kube( + group = "sts.rustfs.com", + version = "v1alpha1", + kind = "PolicyBinding", + namespaced, + status = "PolicyBindingStatus", + shortname = "policybinding", + plural = "policybindings", + singular = "policybinding", + printcolumn = r#"{"name":"State", "type":"string", "jsonPath":".status.currentState"}"#, + printcolumn = r#"{"name":"Age", "type":"date", "jsonPath":".metadata.creationTimestamp"}"#, + crates(serde_json = "k8s_openapi::serde_json") +)] +#[serde(rename_all = "camelCase")] +pub struct PolicyBindingSpec { + pub application: PolicyBindingApplication, + #[x_kube(validation = Rule::new("self.size() > 0").message("policies must contain at least one policy"))] + pub policies: Vec, +} + +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema)] +#[serde(rename_all = "camelCase")] +pub struct PolicyBindingApplication { + pub namespace: String, + pub serviceaccount: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, KubeSchema)] +#[serde(rename_all = "camelCase")] +pub struct PolicyBindingStatus { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_state: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +#[derive(Deserialize, Serialize, Clone, Debug, Default, KubeSchema)] +#[serde(rename_all = "camelCase")] +pub struct PolicyBindingUsage { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authorizations: Option, +}