From 269450c2eb434d5b0eba216f8a8ec397890a78de Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:15:45 +0200 Subject: [PATCH 01/14] chore: add regex and uuid dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These dependencies are required by the framework module that will be added in subsequent commits: - regex: used by the attributed_string_type! macro for compile-time validated string types (e.g. RFC 1035/1123 name validation) - uuid: used for generating unique owner references in cluster resource management No code changes — dependencies are not yet referenced. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 328 ++++++++--- Cargo.nix | 974 +++++++++++++++++++++++++++----- Cargo.toml | 2 + rust/operator-binary/Cargo.toml | 2 + 4 files changed, 1104 insertions(+), 202 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6e8310c..5b96a7ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -830,6 +830,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -977,10 +983,23 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "git2" version = "0.20.4" @@ -1025,9 +1044,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1042,6 +1061,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1050,14 +1078,14 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -1317,6 +1345,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -1336,9 +1370,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1351,7 +1385,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -1369,16 +1405,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1413,9 +1439,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1428,9 +1454,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -1464,9 +1490,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -1476,14 +1502,14 @@ dependencies = [ [[package]] name = "json-patch" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" dependencies = [ "jsonptr", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1664,17 +1690,23 @@ dependencies = [ "spin", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.4+1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "9b26f66f35e1871b22efcf7191564123d2a446ca0538cde63c23adfefa9b15b7" dependencies = [ "cc", "libc", @@ -2062,18 +2094,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" dependencies = [ "proc-macro2", "quote", @@ -2152,6 +2184,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -2233,6 +2275,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.6" @@ -2474,9 +2522,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -2501,9 +2549,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -2906,6 +2954,7 @@ dependencies = [ "futures 0.3.32", "indoc", "product-config", + "regex", "rstest", "serde", "serde_json", @@ -2915,6 +2964,7 @@ dependencies = [ "strum", "tokio", "tracing", + "uuid", ] [[package]] @@ -3292,9 +3342,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3385,9 +3435,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "base64", @@ -3412,9 +3462,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -3442,9 +3492,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ "base64", "bitflags", @@ -3452,13 +3502,13 @@ dependencies = [ "futures-util", "http", "http-body", - "iri-string", "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -3650,6 +3700,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3689,14 +3750,23 @@ version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -3707,9 +3777,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -3717,9 +3787,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3727,9 +3797,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -3740,18 +3810,52 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -3910,19 +4014,107 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.3" @@ -3945,9 +4137,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" diff --git a/Cargo.nix b/Cargo.nix index 5ecf9a54..748147be 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -961,9 +961,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.60"; + version = "1.2.62"; edition = "2018"; - sha256 = "084a8ziprdlyrj865f3303qr0b7aaggilkl18slncss6m4yp1ia3"; + sha256 = "164zsxcy2zzvbbh1qpbrsssz8kmria41j4agih47sal3y1cyip51"; authors = [ "Alex Crichton " ]; @@ -2589,7 +2589,19 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "foldhash" = rec { + "foldhash 0.1.5" = rec { + crateName = "foldhash"; + version = "0.1.5"; + edition = "2021"; + sha256 = "1wisr1xlc2bj7hk4rgkcjkz3j2x4dhd1h9lwk7mj8p71qpdgbi6r"; + authors = [ + "Orson Peters " + ]; + features = { + "default" = [ "std" ]; + }; + }; + "foldhash 0.2.0" = rec { crateName = "foldhash"; version = "0.2.0"; edition = "2021"; @@ -3072,7 +3084,7 @@ rec { } { name = "r-efi"; - packageId = "r-efi"; + packageId = "r-efi 5.3.0"; usesDefaultFeatures = false; target = { target, features }: (("uefi" == target."os" or null) && ("efi_rng" == target."getrandom_backend" or null)); } @@ -3088,6 +3100,90 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "getrandom 0.4.2" = rec { + crateName = "getrandom"; + version = "0.4.2"; + edition = "2024"; + sha256 = "0mb5833hf9pvn9dhvxjgfg5dx0m77g8wavvjdpvpnkp9fil1xr8d"; + authors = [ + "The Rand Project Developers" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ((("linux" == target."os" or null) || ("android" == target."os" or null)) && (!((("linux" == target."os" or null) && ("" == target."env" or null)) || ("custom" == target."getrandom_backend" or null) || ("linux_raw" == target."getrandom_backend" or null) || ("rdrand" == target."getrandom_backend" or null) || ("rndr" == target."getrandom_backend" or null)))); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("dragonfly" == target."os" or null) || ("freebsd" == target."os" or null) || ("hurd" == target."os" or null) || ("illumos" == target."os" or null) || ("cygwin" == target."os" or null) || (("horizon" == target."os" or null) && ("arm" == target."arch" or null))); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("haiku" == target."os" or null) || ("redox" == target."os" or null) || ("nto" == target."os" or null) || ("aix" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("ios" == target."os" or null) || ("visionos" == target."os" or null) || ("watchos" == target."os" or null) || ("tvos" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("macos" == target."os" or null) || ("openbsd" == target."os" or null) || ("vita" == target."os" or null) || ("emscripten" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ("netbsd" == target."os" or null); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ("solaris" == target."os" or null); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ("vxworks" == target."os" or null); + } + { + name = "r-efi"; + packageId = "r-efi 6.0.0"; + usesDefaultFeatures = false; + target = { target, features }: (("uefi" == target."os" or null) && ("efi_rng" == target."getrandom_backend" or null)); + } + { + name = "wasip2"; + packageId = "wasip2"; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && ("wasi" == target."os" or null) && ("p2" == target."env" or null)); + } + { + name = "wasip3"; + packageId = "wasip3"; + target = { target, features }: (("wasm32" == target."arch" or null) && ("wasi" == target."os" or null) && ("p3" == target."env" or null)); + } + ]; + features = { + "sys_rng" = [ "dep:rand_core" ]; + "wasm_js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + }; + }; "git2" = rec { crateName = "git2"; version = "0.20.4"; @@ -3214,9 +3310,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.13"; + version = "0.4.14"; edition = "2021"; - sha256 = "0m6w5gg0n0m1m5915bxrv8n4rlazhx5icknkslz719jhh4xdli1g"; + sha256 = "0cw7jk7kn2vn6f8w8ssh6gis1mljnfjxd606gvi4sjpyjayfy7qp"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3284,6 +3380,36 @@ rec { features = { }; }; + "hashbrown 0.15.5" = rec { + crateName = "hashbrown"; + version = "0.15.5"; + edition = "2021"; + sha256 = "189qaczmjxnikm9db748xyhiw04kpmhm9xj9k9hg0sgx7pjwyacj"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "foldhash"; + packageId = "foldhash 0.1.5"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "dep:alloc" ]; + "allocator-api2" = [ "dep:allocator-api2" ]; + "core" = [ "dep:core" ]; + "default" = [ "default-hasher" "inline-more" "allocator-api2" "equivalent" "raw-entry" ]; + "default-hasher" = [ "dep:foldhash" ]; + "equivalent" = [ "dep:equivalent" ]; + "nightly" = [ "bumpalo/allocator_api" ]; + "rayon" = [ "dep:rayon" ]; + "rustc-dep-of-std" = [ "nightly" "core" "alloc" "rustc-internal-api" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "default-hasher" ]; + }; "hashbrown 0.16.1" = rec { crateName = "hashbrown"; version = "0.16.1"; @@ -3308,7 +3434,7 @@ rec { } { name = "foldhash"; - packageId = "foldhash"; + packageId = "foldhash 0.2.0"; optional = true; usesDefaultFeatures = false; } @@ -3327,14 +3453,11 @@ rec { }; resolvedDefaultFeatures = [ "allocator-api2" "default" "default-hasher" "equivalent" "inline-more" "raw-entry" ]; }; - "hashbrown 0.17.0" = rec { + "hashbrown 0.17.1" = rec { crateName = "hashbrown"; - version = "0.17.0"; + version = "0.17.1"; edition = "2024"; - sha256 = "0l8gvcz80lvinb7x22h53cqbi2y1fm603y2jhhh9qwygvkb7sijg"; - authors = [ - "Amanieu d'Antras " - ]; + sha256 = "0jmqz7i4yl6cm7rbn0i2ffkfrmwi6xkmzkaldr2v8bcsx2v0jngd"; features = { "alloc" = [ "dep:alloc" ]; "allocator-api2" = [ "dep:allocator-api2" ]; @@ -4259,6 +4382,22 @@ rec { }; resolvedDefaultFeatures = [ "baked" ]; }; + "id-arena" = rec { + crateName = "id-arena"; + version = "2.3.0"; + edition = "2021"; + sha256 = "0m6rs0jcaj4mg33gkv98d71w3hridghp5c4yr928hplpkgbnfc1x"; + libName = "id_arena"; + authors = [ + "Nick Fitzgerald " + "Aleksey Kladov " + ]; + features = { + "default" = [ "std" ]; + "rayon" = [ "dep:rayon" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "ident_case" = rec { crateName = "ident_case"; version = "1.0.1"; @@ -4301,9 +4440,9 @@ rec { }; "idna_adapter" = rec { crateName = "idna_adapter"; - version = "1.2.1"; - edition = "2021"; - sha256 = "0i0339pxig6mv786nkqcxnwqa87v4m94b2653f6k3aj0jmhfkjis"; + version = "1.2.2"; + edition = "2024"; + sha256 = "0557p76l8hj35r9zn1yv7c6x1c0qbrsffmg80n0yy8361ly3fs6b"; authors = [ "The rust-url developers" ]; @@ -4337,10 +4476,31 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown 0.17.0"; + packageId = "hashbrown 0.17.1"; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: false; + } + { + name = "serde_core"; + packageId = "serde_core"; + optional = true; usesDefaultFeatures = false; } ]; + devDependencies = [ + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; features = { "arbitrary" = [ "dep:arbitrary" ]; "borsh" = [ "dep:borsh" ]; @@ -4350,7 +4510,7 @@ rec { "serde" = [ "dep:serde_core" "dep:serde" ]; "sval" = [ "dep:sval" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; + resolvedDefaultFeatures = [ "default" "serde" "std" ]; }; "indoc" = rec { crateName = "indoc"; @@ -4395,39 +4555,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "iri-string" = rec { - crateName = "iri-string"; - version = "0.7.12"; - edition = "2021"; - sha256 = "082fpx6c5ghvmqpwxaf2b268m47z2ic3prajqbmi1s1qpfj5kri5"; - libName = "iri_string"; - authors = [ - "YOSHIOKA Takuma " - ]; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - ]; - features = { - "alloc" = [ "serde?/alloc" ]; - "default" = [ "std" ]; - "memchr" = [ "dep:memchr" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" "memchr?/std" "serde?/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "is_terminal_polyfill" = rec { crateName = "is_terminal_polyfill"; version = "1.70.2"; @@ -4497,9 +4624,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.23"; + version = "0.2.24"; edition = "2021"; - sha256 = "0nc37n7jvgrzxdkcgc2hsfdf70lfagigjalh4igjrm5njvf4cd8s"; + sha256 = "0g87al8yqp05m63dhqzi359xgsslc0grqz00nvfdyq8dcayms2zh"; authors = [ "Andrew Gallant " ]; @@ -4579,9 +4706,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.23"; + version = "0.2.24"; edition = "2021"; - sha256 = "192ss3cnixvg79cpa76clwkhn4mmz10vnwsbf7yjw8i484s8p31a"; + sha256 = "1mz6v0d1hd8wjgfzccgda5g9z01s1yxnyiizvahjw0pq1w1xw070"; procMacro = true; libName = "jiff_static"; authors = [ @@ -4661,9 +4788,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.95"; + version = "0.3.98"; edition = "2021"; - sha256 = "1jhj3kgxxgwm0cpdjiz7i2qapqr7ya9qswadmr63dhwx3lnyjr19"; + sha256 = "024zjwpxp6fri4j79bh1686q1x4nw4a06fh1a28zv2rzc4973pv7"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4672,7 +4799,6 @@ rec { { name = "cfg-if"; packageId = "cfg-if"; - optional = true; } { name = "futures-util"; @@ -4694,17 +4820,16 @@ rec { ]; features = { "default" = [ "std" "unsafe-eval" ]; - "futures" = [ "dep:cfg-if" "dep:futures-util" ]; - "futures-core-03-stream" = [ "futures" "dep:futures-core" ]; - "std" = [ "wasm-bindgen/std" ]; + "futures-core-03-stream" = [ "dep:futures-util" "dep:futures-core" ]; + "std" = [ "wasm-bindgen/std" "dep:futures-util" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "std" "unsafe-eval" ]; + resolvedDefaultFeatures = [ "default" "std" "unsafe-eval" ]; }; "json-patch" = rec { crateName = "json-patch"; - version = "4.1.0"; + version = "4.2.0"; edition = "2021"; - sha256 = "147yaxmv3i4s0bdna86rgwpmqh2507fn4ighfpplaiqkw8ay807k"; + sha256 = "0wkv896d0pzq56i2kkl0giqpv117fwvhbpgs8iz85805w66l68bl"; libName = "json_patch"; authors = [ "Ivan Dubrov " @@ -4725,7 +4850,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 1.0.69"; + packageId = "thiserror 2.0.18"; } ]; devDependencies = [ @@ -5516,11 +5641,23 @@ rec { }; resolvedDefaultFeatures = [ "spin" "spin_no_std" ]; }; + "leb128fmt" = rec { + crateName = "leb128fmt"; + version = "0.1.0"; + edition = "2021"; + sha256 = "1chxm1484a0bly6anh6bd7a99sn355ymlagnwj3yajafnpldkv89"; + authors = [ + "Bryant Luk " + ]; + features = { + "default" = [ "std" ]; + }; + }; "libc" = rec { crateName = "libc"; - version = "0.2.185"; + version = "0.2.186"; edition = "2021"; - sha256 = "13rbdaa59l3w92q7kfcxx8zbikm99zzw54h59aqvcv5wx47jrzsj"; + sha256 = "0rnyhzjyqq9x56skkllbjzzzwym3r61lq3l4hqj64v71gw0r3av8"; authors = [ "The Rust Project Developers" ]; @@ -5534,10 +5671,10 @@ rec { }; "libgit2-sys" = rec { crateName = "libgit2-sys"; - version = "0.18.3+1.9.2"; + version = "0.18.4+1.9.3"; edition = "2021"; links = "git2"; - sha256 = "11rlbyihj3k35mnkxxz4yvsnlx33a4r9srl66c5vp08pp72arcy9"; + sha256 = "1dqmkgxgxb937kkcsf05r93a9li385b92wfgxwi1p1z16mpzc9lv"; libName = "libgit2_sys"; libPath = "lib.rs"; authors = [ @@ -6882,9 +7019,9 @@ rec { }; "pin-project" = rec { crateName = "pin-project"; - version = "1.1.11"; + version = "1.1.12"; edition = "2021"; - sha256 = "05zm3y3bl83ypsr6favxvny2kys4i19jiz1y18ylrbxwsiz9qx7i"; + sha256 = "1sbcs3s240z2w4jaga53c3jl5maw4qprf0a9kfcagcq0h7kdkw6b"; libName = "pin_project"; dependencies = [ { @@ -6896,9 +7033,9 @@ rec { }; "pin-project-internal" = rec { crateName = "pin-project-internal"; - version = "1.1.11"; + version = "1.1.12"; edition = "2021"; - sha256 = "1ik4mpb92da75inmjvxf2qm61vrnwml3x24wddvrjlqh1z9hxcnr"; + sha256 = "12a3c85sa005ahk1qm673h1akx2fa8qfvpb0ybd5aj788cpy5459"; procMacro = true; libName = "pin_project_internal"; dependencies = [ @@ -7100,6 +7237,45 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; + "prettyplease" = rec { + crateName = "prettyplease"; + version = "0.2.37"; + edition = "2021"; + links = "prettyplease02"; + sha256 = "0azn11i1kh0byabhsgab6kqs74zyrg69xkirzgqyhz6xmjnsi727"; + authors = [ + "David Tolnay " + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + usesDefaultFeatures = false; + } + { + name = "syn"; + packageId = "syn 2.0.117"; + usesDefaultFeatures = false; + features = [ "full" ]; + } + ]; + devDependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + usesDefaultFeatures = false; + } + { + name = "syn"; + packageId = "syn 2.0.117"; + usesDefaultFeatures = false; + features = [ "clone-impls" "extra-traits" "parsing" "printing" "visit-mut" ]; + } + ]; + features = { + "verbatim" = [ "syn/parsing" ]; + }; + }; "primeorder" = rec { crateName = "primeorder"; version = "0.13.6"; @@ -7306,7 +7482,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "proc-macro" ]; }; - "r-efi" = rec { + "r-efi 5.3.0" = rec { crateName = "r-efi"; version = "5.3.0"; edition = "2018"; @@ -7318,6 +7494,17 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; + "r-efi 6.0.0" = rec { + crateName = "r-efi"; + version = "6.0.0"; + edition = "2018"; + sha256 = "1gyrl2k5fyzj9k7kchg2n296z5881lg7070msabid09asp3wkp7q"; + libName = "r_efi"; + features = { + "core" = [ "dep:core" ]; + "rustc-dep-of-std" = [ "core" ]; + }; + }; "rand 0.8.6" = rec { crateName = "rand"; version = "0.8.6"; @@ -8250,9 +8437,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.38"; + version = "0.23.40"; edition = "2021"; - sha256 = "089ssmhd79f0kd22brh6lkaadql2p3pi6579ax1s0kn1n9pldyb9"; + sha256 = "12qnv3ag4wrw7aj8jng74kgrilpjm2b1rfcjaac8h691frccv1pg"; dependencies = [ { name = "log"; @@ -8350,9 +8537,9 @@ rec { }; "rustls-pki-types" = rec { crateName = "rustls-pki-types"; - version = "1.14.0"; + version = "1.14.1"; edition = "2021"; - sha256 = "1p9zsgslvwzzkzhm6bqicffqndr4jpx67992b0vl0pi21a5hy15y"; + sha256 = "1a9pr54y0f3qr97bxpd3ahjldq0gqdld0h799xbnwdzbwxx1k9rh"; libName = "rustls_pki_types"; dependencies = [ { @@ -9579,6 +9766,10 @@ rec { name = "product-config"; packageId = "product-config"; } + { + name = "regex"; + packageId = "regex"; + } { name = "serde"; packageId = "serde"; @@ -9615,6 +9806,11 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "uuid"; + packageId = "uuid"; + features = [ "v4" ]; + } ]; buildDependencies = [ { @@ -10914,9 +11110,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.1"; + version = "1.52.3"; edition = "2021"; - sha256 = "1imw1dkkv38p66i33m5hsyk3d6prsbyrayjvqhndjvz89ybywzdn"; + sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; authors = [ "Tokio Contributors " ]; @@ -11281,9 +11477,9 @@ rec { }; "tonic" = rec { crateName = "tonic"; - version = "0.14.5"; - edition = "2021"; - sha256 = "1v4k7aa28m7722gz9qak2jiy7lis1ycm4fdmq63iip4m0qdcdizy"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1vs5ci6z6b9xhfsnx4s8qx6bqi1zzcrxncjp71147a0gqwc5aamc"; authors = [ "Lucio Franco " ]; @@ -11410,9 +11606,9 @@ rec { }; "tonic-prost" = rec { crateName = "tonic-prost"; - version = "0.14.5"; - edition = "2021"; - sha256 = "02fkg2bv87q0yds2wz3w0s7i1x6qcgbrl00dy6ipajdapfh7clx5"; + version = "0.14.6"; + edition = "2024"; + sha256 = "184y40nf0iyzc5rg32ivgd88snv68sqy1kchynn55r1vhml9z12h"; libName = "tonic_prost"; authors = [ "Lucio Franco " @@ -11554,9 +11750,9 @@ rec { }; "tower-http" = rec { crateName = "tower-http"; - version = "0.6.8"; + version = "0.6.10"; edition = "2018"; - sha256 = "1y514jwzbyrmrkbaajpwmss4rg0mak82k16d6588w9ncaffmbrnl"; + sha256 = "0lfbddgrhmxhnb3afazawsl4cxqfcs8wvq5hm34ija0wz3czvmk8"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -11590,11 +11786,6 @@ rec { packageId = "http-body"; optional = true; } - { - name = "iri-string"; - packageId = "iri-string"; - optional = true; - } { name = "mime"; packageId = "mime"; @@ -11624,6 +11815,11 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "url"; + packageId = "url"; + optional = true; + } ]; devDependencies = [ { @@ -11645,35 +11841,33 @@ rec { } ]; features = { - "async-compression" = [ "dep:async-compression" ]; "auth" = [ "base64" "validate-request" ]; "base64" = [ "dep:base64" ]; "catch-panic" = [ "tracing" "futures-util/std" "dep:http-body" "dep:http-body-util" ]; - "compression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; "compression-full" = [ "compression-br" "compression-deflate" "compression-gzip" "compression-zstd" ]; - "compression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "decompression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "compression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "decompression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; "decompression-full" = [ "decompression-br" "decompression-deflate" "decompression-gzip" "decompression-zstd" ]; - "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; - "fs" = [ "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; - "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; + "decompression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "follow-redirect" = [ "futures-util" "dep:http-body" "dep:url" "tower/util" ]; + "fs" = [ "dep:tokio" "tokio?/fs" "tokio?/io-util" "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio-util/io" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" ]; + "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "on-early-drop" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; "httpdate" = [ "dep:httpdate" ]; - "iri-string" = [ "dep:iri-string" ]; "limit" = [ "dep:http-body" "dep:http-body-util" ]; - "metrics" = [ "dep:http-body" "tokio/time" ]; + "metrics" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "mime" = [ "dep:mime" ]; "mime_guess" = [ "dep:mime_guess" ]; + "on-early-drop" = [ "dep:http-body" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "request-id" = [ "uuid" ]; - "timeout" = [ "dep:http-body" "tokio/time" ]; - "tokio" = [ "dep:tokio" ]; + "timeout" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "tokio-util" = [ "dep:tokio-util" ]; "tower" = [ "dep:tower" ]; "trace" = [ "dep:http-body" "tracing" ]; @@ -11682,7 +11876,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "iri-string" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -12293,12 +12487,78 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "valuable" = rec { - crateName = "valuable"; - version = "0.1.1"; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.1"; edition = "2021"; - sha256 = "0r9srp55v7g27s5bg7a2m095fzckrcdca5maih6dy9bay6fflwxs"; - features = { + sha256 = "0xlwg23rmsfl3gx98qsyzpl24pf4bs9wi3mqx5c6i319hyb4mmyx"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "getrandom"; + packageId = "getrandom 0.4.2"; + optional = true; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); + } + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "rng" "std" "v4" ]; + }; + "valuable" = rec { + crateName = "valuable"; + version = "0.1.1"; + edition = "2021"; + sha256 = "0r9srp55v7g27s5bg7a2m095fzckrcdca5maih6dy9bay6fflwxs"; + features = { "default" = [ "std" ]; "derive" = [ "valuable-derive" ]; "std" = [ "alloc" ]; @@ -12366,7 +12626,7 @@ rec { dependencies = [ { name = "wit-bindgen"; - packageId = "wit-bindgen"; + packageId = "wit-bindgen 0.57.1"; usesDefaultFeatures = false; } ]; @@ -12378,11 +12638,36 @@ rec { "rustc-dep-of-std" = [ "core" "alloc" "wit-bindgen/rustc-dep-of-std" ]; }; }; + "wasip3" = rec { + crateName = "wasip3"; + version = "0.4.0+wasi-0.3.0-rc-2026-01-06"; + edition = "2021"; + sha256 = "19dc8p0y2mfrvgk3qw3c3240nfbylv22mvyxz84dqpgai2zzha2l"; + dependencies = [ + { + name = "wit-bindgen"; + packageId = "wit-bindgen 0.51.0"; + usesDefaultFeatures = false; + features = [ "async" ]; + } + ]; + devDependencies = [ + { + name = "wit-bindgen"; + packageId = "wit-bindgen 0.51.0"; + usesDefaultFeatures = false; + features = [ "async-spawn" ]; + } + ]; + features = { + "http-compat" = [ "dep:bytes" "dep:http-body" "dep:http" "dep:thiserror" "wit-bindgen/async-spawn" ]; + }; + }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.118"; + version = "0.2.121"; edition = "2021"; - sha256 = "129s5r14fx4v4xrzpx2c6l860nkxpl48j50y7kl6j16bpah3iy8b"; + sha256 = "14375vc40l67lk9rxp59my4r6s64h2an3vjfh9j0hnqngk8f3b29"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12431,9 +12716,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.68"; + version = "0.4.71"; edition = "2021"; - sha256 = "1y7bq5d9fk7s9xaayx38bgs9ns35na0kpb5zw19944zvya1x6wgk"; + sha256 = "1f3k8r13nqshrlxwq0naxpbh250b4l6p526wlw2m78pv7w6jsjcn"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12443,7 +12728,6 @@ rec { name = "js-sys"; packageId = "js-sys"; usesDefaultFeatures = false; - features = [ "futures" ]; } { name = "wasm-bindgen"; @@ -12460,9 +12744,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.118"; + version = "0.2.121"; edition = "2021"; - sha256 = "1v98r8vs17cj8918qsg0xx4nlg4nxk1g0jd4nwnyrh1687w29zzf"; + sha256 = "0y45ghbkvs5rmxvdyhqrx8nzyy45rdx6619c01iaarykmzsfcs4f"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12484,9 +12768,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.118"; + version = "0.2.121"; edition = "2021"; - sha256 = "0169jr0q469hfx5zqxfyywf2h2f4aj17vn4zly02nfwqmxghc24x"; + sha256 = "1wjr69qa8rwmk4v7243dr100k393qi0avznk6p5sgck4bk1rwnnr"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12520,10 +12804,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.118"; + version = "0.2.121"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "0ag1vvdzi4334jlzilsy14y3nyzwddf1ndn62fyhf6bg62g4vl2z"; + sha256 = "0h9la4176j5bvgbr64cqkmirif8z59vrcax9i4qx1w79045i1q64"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12536,11 +12820,121 @@ rec { ]; }; + "wasm-encoder" = rec { + crateName = "wasm-encoder"; + version = "0.244.0"; + edition = "2021"; + sha256 = "06c35kv4h42vk3k51xjz1x6hn3mqwfswycmr6ziky033zvr6a04r"; + libName = "wasm_encoder"; + authors = [ + "Nick Fitzgerald " + ]; + dependencies = [ + { + name = "leb128fmt"; + packageId = "leb128fmt"; + usesDefaultFeatures = false; + } + { + name = "wasmparser"; + packageId = "wasmparser"; + optional = true; + usesDefaultFeatures = false; + features = [ "simd" "simd" ]; + } + ]; + features = { + "component-model" = [ "wasmparser?/component-model" ]; + "default" = [ "std" "component-model" ]; + "std" = [ "wasmparser?/std" ]; + "wasmparser" = [ "dep:wasmparser" ]; + }; + resolvedDefaultFeatures = [ "component-model" "std" "wasmparser" ]; + }; + "wasm-metadata" = rec { + crateName = "wasm-metadata"; + version = "0.244.0"; + edition = "2021"; + sha256 = "02f9dhlnryd2l7zf03whlxai5sv26x4spfibjdvc3g9gd8z3a3mv"; + libName = "wasm_metadata"; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "indexmap"; + packageId = "indexmap"; + usesDefaultFeatures = false; + features = [ "serde" ]; + } + { + name = "wasm-encoder"; + packageId = "wasm-encoder"; + usesDefaultFeatures = false; + features = [ "std" "component-model" ]; + } + { + name = "wasmparser"; + packageId = "wasmparser"; + usesDefaultFeatures = false; + features = [ "simd" "std" "component-model" "hash-collections" ]; + } + ]; + features = { + "clap" = [ "dep:clap" ]; + "default" = [ "oci" "serde" ]; + "oci" = [ "dep:auditable-serde" "dep:flate2" "dep:url" "dep:spdx" "dep:serde_json" "serde" ]; + "serde" = [ "dep:serde_derive" "dep:serde" ]; + }; + }; + "wasmparser" = rec { + crateName = "wasmparser"; + version = "0.244.0"; + edition = "2021"; + sha256 = "1zi821hrlsxfhn39nqpmgzc0wk7ax3dv6vrs5cw6kb0v5v3hgf27"; + authors = [ + "Yury Delendik " + ]; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags"; + } + { + name = "hashbrown"; + packageId = "hashbrown 0.15.5"; + optional = true; + usesDefaultFeatures = false; + features = [ "default-hasher" ]; + } + { + name = "indexmap"; + packageId = "indexmap"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "semver"; + packageId = "semver"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "component-model" = [ "dep:semver" ]; + "default" = [ "std" "validate" "serde" "features" "component-model" "hash-collections" "simd" ]; + "hash-collections" = [ "dep:hashbrown" "dep:indexmap" ]; + "serde" = [ "dep:serde" "indexmap?/serde" "hashbrown?/serde" ]; + "std" = [ "indexmap?/std" ]; + }; + resolvedDefaultFeatures = [ "component-model" "features" "hash-collections" "simd" "std" "validate" ]; + }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.95"; + version = "0.3.98"; edition = "2021"; - sha256 = "0zfr2jy5bpkkggl88i43yy37p538hg20i56kwn421yj9g6qznbag"; + sha256 = "1aijiwx7wsfzj37p1gnqn6wv4j2ppf4rqwhrzb8blf6gigzjsmsb"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -12624,6 +13018,7 @@ rec { "CssStyleSheet" = [ "StyleSheet" ]; "CssSupportsRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; "CssTransition" = [ "Animation" "EventTarget" ]; + "CssViewTransitionRule" = [ "CssRule" ]; "CustomEvent" = [ "Event" ]; "DedicatedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; "DelayNode" = [ "AudioNode" "EventTarget" ]; @@ -13837,9 +14232,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.1"; + version = "1.0.2"; edition = "2021"; - sha256 = "1dbji1bwviy08pl74f2qw2m4w9hc4p3vyl3lfj05jdydy59w1nh9"; + sha256 = "1l7xnfvlgy4da6gq5ip2bgcm8i9d0rwzaxg1p88nlw8lxy5p1q9f"; dependencies = [ { name = "memchr"; @@ -13860,7 +14255,34 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "ascii" "binary" "default" "parser" "std" ]; }; - "wit-bindgen" = rec { + "wit-bindgen 0.51.0" = rec { + crateName = "wit-bindgen"; + version = "0.51.0"; + edition = "2024"; + sha256 = "19fazgch8sq5cvjv3ynhhfh5d5x08jq2pkw8jfb05vbcyqcr496p"; + libName = "wit_bindgen"; + authors = [ + "Alex Crichton " + ]; + dependencies = [ + { + name = "wit-bindgen-rust-macro"; + packageId = "wit-bindgen-rust-macro"; + optional = true; + } + ]; + features = { + "async" = [ "std" "wit-bindgen-rust-macro?/async" ]; + "async-spawn" = [ "async" "dep:futures" ]; + "bitflags" = [ "dep:bitflags" ]; + "default" = [ "macros" "realloc" "async" "std" "bitflags" ]; + "inter-task-wakeup" = [ "async" ]; + "macros" = [ "dep:wit-bindgen-rust-macro" ]; + "rustc-dep-of-std" = [ "dep:core" "dep:alloc" ]; + }; + resolvedDefaultFeatures = [ "async" "std" ]; + }; + "wit-bindgen 0.57.1" = rec { crateName = "wit-bindgen"; version = "0.57.1"; edition = "2024"; @@ -13880,6 +14302,290 @@ rec { "rustc-dep-of-std" = [ "dep:core" "dep:alloc" ]; }; }; + "wit-bindgen-core" = rec { + crateName = "wit-bindgen-core"; + version = "0.51.0"; + edition = "2024"; + sha256 = "1p2jszqsqbx8k7y8nwvxg65wqzxjm048ba5phaq8r9iy9ildwqga"; + libName = "wit_bindgen_core"; + authors = [ + "Alex Crichton " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "heck"; + packageId = "heck"; + } + { + name = "wit-parser"; + packageId = "wit-parser"; + } + ]; + features = { + "clap" = [ "dep:clap" ]; + "serde" = [ "dep:serde" ]; + }; + }; + "wit-bindgen-rust" = rec { + crateName = "wit-bindgen-rust"; + version = "0.51.0"; + edition = "2024"; + sha256 = "08bzn5fsvkb9x9wyvyx98qglknj2075xk1n7c5jxv15jykh6didp"; + libName = "wit_bindgen_rust"; + authors = [ + "Alex Crichton " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "heck"; + packageId = "heck"; + } + { + name = "indexmap"; + packageId = "indexmap"; + } + { + name = "prettyplease"; + packageId = "prettyplease"; + } + { + name = "syn"; + packageId = "syn 2.0.117"; + features = [ "printing" ]; + } + { + name = "wasm-metadata"; + packageId = "wasm-metadata"; + usesDefaultFeatures = false; + } + { + name = "wit-bindgen-core"; + packageId = "wit-bindgen-core"; + } + { + name = "wit-component"; + packageId = "wit-component"; + } + ]; + features = { + "clap" = [ "dep:clap" "wit-bindgen-core/clap" ]; + "serde" = [ "dep:serde" "wit-bindgen-core/serde" ]; + }; + }; + "wit-bindgen-rust-macro" = rec { + crateName = "wit-bindgen-rust-macro"; + version = "0.51.0"; + edition = "2024"; + sha256 = "0ymizapzv2id89igxsz2n587y2hlfypf6n8kyp68x976fzyrn3qc"; + procMacro = true; + libName = "wit_bindgen_rust_macro"; + authors = [ + "Alex Crichton " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "prettyplease"; + packageId = "prettyplease"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.117"; + features = [ "printing" ]; + } + { + name = "wit-bindgen-core"; + packageId = "wit-bindgen-core"; + } + { + name = "wit-bindgen-rust"; + packageId = "wit-bindgen-rust"; + } + ]; + features = { + }; + resolvedDefaultFeatures = [ "async" ]; + }; + "wit-component" = rec { + crateName = "wit-component"; + version = "0.244.0"; + edition = "2021"; + sha256 = "1clwxgsgdns3zj2fqnrjcp8y5gazwfa1k0sy5cbk0fsmx4hflrlx"; + libName = "wit_component"; + authors = [ + "Peter Huene " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "bitflags"; + packageId = "bitflags"; + } + { + name = "indexmap"; + packageId = "indexmap"; + usesDefaultFeatures = false; + } + { + name = "log"; + packageId = "log"; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "serde_derive"; + packageId = "serde_derive"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "wasm-encoder"; + packageId = "wasm-encoder"; + usesDefaultFeatures = false; + features = [ "std" "wasmparser" ]; + } + { + name = "wasm-metadata"; + packageId = "wasm-metadata"; + usesDefaultFeatures = false; + } + { + name = "wasmparser"; + packageId = "wasmparser"; + usesDefaultFeatures = false; + features = [ "simd" "std" "component-model" "simd" ]; + } + { + name = "wit-parser"; + packageId = "wit-parser"; + features = [ "decoding" "serde" ]; + } + ]; + devDependencies = [ + { + name = "wasm-metadata"; + packageId = "wasm-metadata"; + usesDefaultFeatures = false; + features = [ "oci" ]; + } + { + name = "wasmparser"; + packageId = "wasmparser"; + usesDefaultFeatures = false; + features = [ "simd" "std" "component-model" "features" ]; + } + ]; + features = { + "dummy-module" = [ "dep:wat" ]; + "semver-check" = [ "dummy-module" ]; + "wat" = [ "dep:wast" "dep:wat" ]; + }; + }; + "wit-parser" = rec { + crateName = "wit-parser"; + version = "0.244.0"; + edition = "2021"; + sha256 = "0dm7avvdxryxd5b02l0g5h6933z1cw5z0d4wynvq2cywq55srj7c"; + libName = "wit_parser"; + authors = [ + "Alex Crichton " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "id-arena"; + packageId = "id-arena"; + } + { + name = "indexmap"; + packageId = "indexmap"; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "log"; + packageId = "log"; + } + { + name = "semver"; + packageId = "semver"; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "serde_derive"; + packageId = "serde_derive"; + optional = true; + } + { + name = "serde_json"; + packageId = "serde_json"; + optional = true; + } + { + name = "unicode-xid"; + packageId = "unicode-xid"; + } + { + name = "wasmparser"; + packageId = "wasmparser"; + optional = true; + usesDefaultFeatures = false; + features = [ "simd" "std" "validate" "component-model" "features" ]; + } + ]; + devDependencies = [ + { + name = "serde_json"; + packageId = "serde_json"; + } + ]; + features = { + "decoding" = [ "dep:wasmparser" ]; + "default" = [ "serde" "decoding" ]; + "serde" = [ "dep:serde" "dep:serde_derive" "indexmap/serde" "serde_json" ]; + "serde_json" = [ "dep:serde_json" ]; + "wat" = [ "decoding" "dep:wat" ]; + }; + resolvedDefaultFeatures = [ "decoding" "default" "serde" "serde_json" ]; + }; "writeable" = rec { crateName = "writeable"; version = "0.6.3"; @@ -13951,9 +14657,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" diff --git a/Cargo.toml b/Cargo.toml index d04c1f1e..119abf53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" +regex = "1" rstest = "0.26" semver = "1.0" serde = { version = "1.0", features = ["derive"] } @@ -32,6 +33,7 @@ snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" +uuid = { version = "1.16", features = ["v4"] } [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 2734bb4e..095324c8 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -21,10 +21,12 @@ indoc.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true +regex.workspace = true snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +uuid.workspace = true [build-dependencies] built.workspace = true From c7f09e9c3c567ead811e21fd1f020023c83a7a61 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:27:17 +0200 Subject: [PATCH 02/14] feat: add framework macros for validated string types Add the attributed_string_type! macro which generates strongly-typed wrappers around strings with compile-time validation attributes: - RFC 1035 label names, RFC 1123 label/subdomain names - Label value constraints, length limits, regex patterns - Automatic FromStr, Display, Serialize/Deserialize, JsonSchema impls - Compile-time constant checks (IS_RFC_1035_LABEL_NAME, MAX_LENGTH, etc.) Also adds a small constant! helper macro for validated const values. The NameIsValidLabelValue trait is included in framework.rs as the macro references it for label-value-safe types. These macros are the foundation for the validated type system that replaces stringly-typed names throughout the operator. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/framework.rs | 28 + rust/operator-binary/src/framework/macros.rs | 2 + .../macros/attributed_string_type.rs | 927 ++++++++++++++++++ .../src/framework/macros/constant.rs | 17 + rust/operator-binary/src/main.rs | 1 + 5 files changed, 975 insertions(+) create mode 100644 rust/operator-binary/src/framework.rs create mode 100644 rust/operator-binary/src/framework/macros.rs create mode 100644 rust/operator-binary/src/framework/macros/attributed_string_type.rs create mode 100644 rust/operator-binary/src/framework/macros/constant.rs diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..0115ccc3 --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,28 @@ +//! Additions to stackable-operator +//! +//! Functions in stackable-operator usually accept generic types like strings and validate the +//! parameters as late as possible. Therefore, nearly all functions have to return a [`Result`] and +//! errors are returned along the call chain. That makes error handling complex because every +//! module re-packages the received error. Also, the validation is repeated if the value is used in +//! different function calls. Sometimes, validation is not necessary if constant values are used, +//! e.g. the name of the operator. +//! +//! This operator uses a different approach. The incoming values are validated as early as possible +//! and wrapped in a fail-safe type. This type is then used along the call chain, validation is not +//! necessary anymore and functions without side effects do not need to return a [`Result`]. +//! +//! However, this operator uses stackable-operator and at the interface, the fail-safe types must +//! be unwrapped and the [`Result`] returned by the stackable-operator function must be handled. +//! This is done by calling [`Result::expect`] which requires thorough testing. +//! +//! When the development of this module has progressed and changes become less frequent, then this +//! module can be incorporated into stackable-operator. The module structure should already +//! resemble the one of stackable-operator. + +#[macro_use] +pub mod macros; + +/// The name is a valid label value +pub trait NameIsValidLabelValue { + fn to_label_value(&self) -> String; +} diff --git a/rust/operator-binary/src/framework/macros.rs b/rust/operator-binary/src/framework/macros.rs new file mode 100644 index 00000000..c25def95 --- /dev/null +++ b/rust/operator-binary/src/framework/macros.rs @@ -0,0 +1,2 @@ +pub mod attributed_string_type; +pub mod constant; diff --git a/rust/operator-binary/src/framework/macros/attributed_string_type.rs b/rust/operator-binary/src/framework/macros/attributed_string_type.rs new file mode 100644 index 00000000..aee756a3 --- /dev/null +++ b/rust/operator-binary/src/framework/macros/attributed_string_type.rs @@ -0,0 +1,927 @@ +use snafu::Snafu; +use strum::{EnumDiscriminants, IntoStaticStr}; + +/// Maximum length of label values +/// +/// Duplicates the private constant [`stackable_operator::kvp::LABEL_VALUE_MAX_LEN`] +pub const MAX_LABEL_VALUE_LENGTH: usize = 63; + +#[derive(Debug, EnumDiscriminants, Snafu)] +#[snafu(visibility(pub))] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("minimum length not met"))] + MinimumLengthNotMet { length: usize, min_length: usize }, + + #[snafu(display("maximum length exceeded"))] + LengthExceeded { length: usize, max_length: usize }, + + #[snafu(display("invalid regular expression"))] + InvalidRegex { source: regex::Error }, + + #[snafu(display("regular expression not matched"))] + RegexNotMatched { value: String, regex: &'static str }, + + #[snafu(display("not a valid label value"))] + InvalidLabelValue { + source: stackable_operator::kvp::LabelValueError, + }, + + #[snafu(display("not a valid label name as defined in RFC 1035"))] + InvalidRfc1035LabelName { + source: stackable_operator::validation::Errors, + }, + + #[snafu(display("not a valid DNS subdomain name as defined in RFC 1123"))] + InvalidRfc1123DnsSubdomainName { + source: stackable_operator::validation::Errors, + }, + + #[snafu(display("not a valid label name as defined in RFC 1123"))] + InvalidRfc1123LabelName { + source: stackable_operator::validation::Errors, + }, + + #[snafu(display("not a valid UUID"))] + InvalidUid { source: uuid::Error }, +} + +/// Helper data type to determine combined regular expressions +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Regex { + /// There is a regular expression but it is unknown (because it was too complicated to + /// calculate it). + Unknown, + + /// `MatchAll` equals `Expression(".*")`, but `MatchAll` can be pattern matched in a const + /// context, whereas `Expression(...)` cannot. + MatchAll, + + /// A regular expression + Expression(&'static str), +} + +impl Regex { + /// Combine this regular expression with the given one. + pub const fn combine(self, other: Regex) -> Regex { + match (self, other) { + (_, Regex::MatchAll) => self, + (Regex::MatchAll, _) => other, + // It is hard to combine two regular expressions and nearly impossible to do this in a + // const context. Fortunately, for most of the data types, only one regular expression + // is set. + _ => Regex::Unknown, + } + } +} + +/// Restricted string type with attributes like maximum length. +/// +/// Fully-qualified types are used to ease the import into other modules. +/// +/// # Examples +/// +/// ```rust +/// attributed_string_type! { +/// ConfigMapName, +/// "The name of a ConfigMap", +/// "airflow-webserver-default", +/// is_rfc_1123_dns_subdomain_name +/// } +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! attributed_string_type { + ($name:ident, $description:literal, $example:literal $(, $attribute:tt)*) => { + #[doc = std::concat!($description, ", e.g. \"", $example, "\"")] + #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] + pub struct $name(String); + + impl $name { + /// The minimum length + pub const MIN_LENGTH: usize = attributed_string_type!(@min_length $($attribute)*); + + /// The maximum length + pub const MAX_LENGTH: usize = attributed_string_type!(@max_length $($attribute)*); + + /// The regular expression + /// + /// This field is not meant to be used outside of this macro. + pub const REGEX: $crate::framework::macros::attributed_string_type::Regex = attributed_string_type!(@regex $($attribute)*); + } + + impl stackable_operator::config::merge::Atomic for $name {} + + impl std::fmt::Display for $name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } + } + + impl From<$name> for String { + fn from(value: $name) -> Self { + value.0 + } + } + + impl From<&$name> for String { + fn from(value: &$name) -> Self { + value.0.clone() + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + + impl std::str::FromStr for $name { + type Err = $crate::framework::macros::attributed_string_type::Error; + + fn from_str(s: &str) -> std::result::Result { + // ResultExt::context is used on most but not all usages of this macro + #[allow(unused_imports)] + use snafu::ResultExt; + + $(attributed_string_type!(@from_str $name, s, $attribute);)* + + Ok(Self(s.to_owned())) + } + } + + impl<'de> serde::Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let string: String = serde::Deserialize::deserialize(deserializer)?; + $name::from_str(&string).map_err(|err| serde::de::Error::custom(&err)) + } + } + + impl serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.serialize(serializer) + } + } + + // The JsonSchema implementation requires `max_length`. + impl stackable_operator::schemars::JsonSchema for $name { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::stringify!($name).into() + } + + fn json_schema(_generator: &mut stackable_operator::schemars::generate::SchemaGenerator) -> stackable_operator::schemars::Schema { + stackable_operator::schemars::json_schema!({ + "type": "string", + "minLength": $name::MIN_LENGTH, + "maxLength": if $name::MAX_LENGTH != usize::MAX { + Some($name::MAX_LENGTH) + } else { + // Do not set maxLength if it is usize::MAX. + None + }, + "pattern": match $name::REGEX { + $crate::framework::macros::attributed_string_type::Regex::Expression(regex) => Some(regex), + _ => None + } + }) + } + } + + impl $name { + /// Converts a string to this type, panicking if the string is invalid. + /// + /// Only use this for compile-time constants or pre-validated values. + pub fn from_str_unsafe(s: &str) -> Self { + std::str::FromStr::from_str(s).expect("should be a valid {name}") + } + } + + #[cfg(test)] + impl $name { + // A dead_code warning is emitted if there is no unit test that calls this function. + pub fn test_example() { + Self::from_str_unsafe($example); + } + } + + $(attributed_string_type!(@trait_impl $name, $attribute);)* + }; + + // std::str::FromStr + + (@from_str $name:ident, $s:expr, (min_length = $min_length:expr)) => { + let length = $s.len() as usize; + snafu::ensure!( + length >= $name::MIN_LENGTH, + $crate::framework::macros::attributed_string_type::MinimumLengthNotMetSnafu { + length, + min_length: $name::MIN_LENGTH, + } + ); + }; + (@from_str $name:ident, $s:expr, (max_length = $max_length:expr)) => { + let length = $s.len() as usize; + snafu::ensure!( + length <= $name::MAX_LENGTH, + $crate::framework::macros::attributed_string_type::LengthExceededSnafu { + length, + max_length: $name::MAX_LENGTH, + } + ); + }; + (@from_str $name:ident, $s:expr, (regex = $regex:expr)) => { + let regex = regex::Regex::new($regex).context($crate::framework::macros::attributed_string_type::InvalidRegexSnafu)?; + snafu::ensure!( + regex.is_match($s), + $crate::framework::macros::attributed_string_type::RegexNotMatchedSnafu { + value: $s, + regex: $regex + } + ); + }; + (@from_str $name:ident, $s:expr, is_rfc_1035_label_name) => { + stackable_operator::validation::is_lowercase_rfc_1035_label($s).context($crate::framework::macros::attributed_string_type::InvalidRfc1035LabelNameSnafu)?; + }; + (@from_str $name:ident, $s:expr, is_rfc_1123_dns_subdomain_name) => { + stackable_operator::validation::is_lowercase_rfc_1123_subdomain($s).context($crate::framework::macros::attributed_string_type::InvalidRfc1123DnsSubdomainNameSnafu)?; + }; + (@from_str $name:ident, $s:expr, is_rfc_1123_label_name) => { + stackable_operator::validation::is_lowercase_rfc_1123_label($s).context($crate::framework::macros::attributed_string_type::InvalidRfc1123LabelNameSnafu)?; + }; + (@from_str $name:ident, $s:expr, is_valid_label_value) => { + stackable_operator::kvp::LabelValue::from_str($s).context($crate::framework::macros::attributed_string_type::InvalidLabelValueSnafu)?; + }; + (@from_str $name:ident, $s:expr, is_uid) => { + uuid::Uuid::try_parse($s).context($crate::framework::macros::attributed_string_type::InvalidUidSnafu)?; + }; + + // MIN_LENGTH + + (@min_length) => { + // The minimum String length is 0. + 0 + }; + (@min_length (min_length = $min_length:expr) $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::max( + $min_length, + attributed_string_type!(@min_length $($attribute)*) + ) + }; + (@min_length (max_length = $max_length:expr) $($attribute:tt)*) => { + // max_length has no opinion on the min_length. + attributed_string_type!(@min_length $($attribute)*) + }; + (@min_length (regex = $regex:expr) $($attribute:tt)*) => { + // regex has no influence on the min_length. + attributed_string_type!(@min_length $($attribute)*) + }; + (@min_length is_rfc_1035_label_name $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::max( + 1, + attributed_string_type!(@min_length $($attribute)*) + ) + }; + (@min_length is_rfc_1123_dns_subdomain_name $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::max( + 1, + attributed_string_type!(@min_length $($attribute)*) + ) + }; + (@min_length is_rfc_1123_label_name $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::max( + 1, + attributed_string_type!(@min_length $($attribute)*) + ) + }; + (@min_length is_valid_label_value $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::max( + 1, + attributed_string_type!(@min_length $($attribute)*) + ) + }; + (@min_length is_uid $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::max( + uuid::fmt::Hyphenated::LENGTH, + attributed_string_type!(@min_length $($attribute)*) + ) + }; + + // MAX_LENGTH + + (@max_length) => { + // If there is no other max_length defined, then the upper bound is usize::MAX. + usize::MAX + }; + (@max_length (min_length = $min_length:expr) $($attribute:tt)*) => { + // min_length has no opinion on the max_length. + attributed_string_type!(@max_length $($attribute)*) + }; + (@max_length (max_length = $max_length:expr) $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::min( + $max_length, + attributed_string_type!(@max_length $($attribute)*) + ) + }; + (@max_length (regex = $regex:expr) $($attribute:tt)*) => { + // regex has no influence on the max_length. + attributed_string_type!(@max_length $($attribute)*) + }; + (@max_length is_rfc_1035_label_name $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::min( + stackable_operator::validation::RFC_1035_LABEL_MAX_LENGTH, + attributed_string_type!(@max_length $($attribute)*) + ) + }; + (@max_length is_rfc_1123_dns_subdomain_name $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::min( + stackable_operator::validation::RFC_1123_SUBDOMAIN_MAX_LENGTH, + attributed_string_type!(@max_length $($attribute)*) + ) + }; + (@max_length is_rfc_1123_label_name $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::min( + stackable_operator::validation::RFC_1123_LABEL_MAX_LENGTH, + attributed_string_type!(@max_length $($attribute)*) + ) + }; + (@max_length is_valid_label_value $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::min( + $crate::framework::macros::attributed_string_type::MAX_LABEL_VALUE_LENGTH, + attributed_string_type!(@max_length $($attribute)*) + ) + }; + (@max_length is_uid $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::min( + uuid::fmt::Hyphenated::LENGTH, + attributed_string_type!(@max_length $($attribute)*) + ) + }; + + // REGEX + + (@regex) => { + // Everything is allowed if there is no other regular expression. + $crate::framework::macros::attributed_string_type::Regex::MatchAll + }; + (@regex (min_length = $min_length:expr) $($attribute:tt)*) => { + // min_length has no influence on the regular expression. + attributed_string_type!(@regex $($attribute)*) + }; + (@regex (max_length = $max_length:expr) $($attribute:tt)*) => { + // max_length has no influence on the regular expression. + attributed_string_type!(@regex $($attribute)*) + }; + (@regex (regex = $regex:expr) $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::Regex::Expression($regex) + .combine(attributed_string_type!(@regex $($attribute)*)) + }; + (@regex is_rfc_1035_label_name $($attribute:tt)*) => { + // see https://github.com/kubernetes/kubernetes/blob/v1.35.0/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L228 + $crate::framework::macros::attributed_string_type::Regex::Expression("^[a-z]([-a-z0-9]*[a-z0-9])?$") + .combine(attributed_string_type!(@regex $($attribute)*)) + }; + (@regex is_rfc_1123_dns_subdomain_name $($attribute:tt)*) => { + // see https://github.com/kubernetes/kubernetes/blob/v1.35.0/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L193 + $crate::framework::macros::attributed_string_type::Regex::Expression("^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$") + .combine(attributed_string_type!(@regex $($attribute)*)) + }; + (@regex is_rfc_1123_label_name $($attribute:tt)*) => { + // see https://github.com/kubernetes/kubernetes/blob/v1.35.0/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L163 + $crate::framework::macros::attributed_string_type::Regex::Expression("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") + .combine(attributed_string_type!(@regex $($attribute)*)) + }; + (@regex is_valid_label_value $($attribute:tt)*) => { + // regular expression from stackable_operator::kvp::label::LABEL_VALUE_REGEX + $crate::framework::macros::attributed_string_type::Regex::Expression("^[a-z0-9A-Z]([a-z0-9A-Z-_.]*[a-z0-9A-Z]+)?$") + .combine(attributed_string_type!(@regex $($attribute)*)) + }; + (@regex is_uid $($attribute:tt)*) => { + $crate::framework::macros::attributed_string_type::Regex::Expression("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") + .combine(attributed_string_type!(@regex $($attribute)*)) + }; + + // additional constants and trait implementations + + (@trait_impl $name:ident, (min_length = $max_length:expr)) => { + }; + (@trait_impl $name:ident, (max_length = $max_length:expr)) => { + }; + (@trait_impl $name:ident, (regex = $regex:expr)) => { + }; + (@trait_impl $name:ident, is_rfc_1035_label_name) => { + impl $name { + pub const IS_RFC_1035_LABEL_NAME: bool = true; + pub const IS_RFC_1123_LABEL_NAME: bool = true; + pub const IS_RFC_1123_SUBDOMAIN_NAME: bool = true; + } + }; + (@trait_impl $name:ident, is_rfc_1123_dns_subdomain_name) => { + impl $name { + pub const IS_RFC_1123_SUBDOMAIN_NAME: bool = true; + } + }; + (@trait_impl $name:ident, is_rfc_1123_label_name) => { + impl $name { + pub const IS_RFC_1123_LABEL_NAME: bool = true; + pub const IS_RFC_1123_SUBDOMAIN_NAME: bool = true; + } + }; + (@trait_impl $name:ident, is_valid_label_value) => { + impl $name { + pub const IS_VALID_LABEL_VALUE: bool = true; + } + + impl $crate::framework::NameIsValidLabelValue for $name { + fn to_label_value(&self) -> String { + self.0.clone() + } + } + }; + (@trait_impl $name:ident, is_uid) => { + impl From for $name { + fn from(value: uuid::Uuid) -> Self { + Self(value.to_string()) + } + } + + impl From<&uuid::Uuid> for $name { + fn from(value: &uuid::Uuid) -> Self { + Self(value.to_string()) + } + } + }; +} + +/// Returns the minimum of the given values. +/// +/// As opposed to [`std::cmp::min`], this function can be used at compile-time. +/// +/// # Examples +/// +/// ```rust +/// assert_eq!(2, min(2, 3)); +/// assert_eq!(4, min(5, 4)); +/// assert_eq!(1, min(1, 1)); +/// ``` +pub const fn min(x: usize, y: usize) -> usize { + if x < y { x } else { y } +} + +/// Returns the maximum of the given values. +/// +/// As opposed to [`std::cmp::max`], this function can be used at compile-time. +/// +/// # Examples +/// +/// ```rust +/// assert_eq!(3, max(2, 3)); +/// assert_eq!(5, max(5, 4)); +/// assert_eq!(1, max(1, 1)); +/// ``` +pub const fn max(x: usize, y: usize) -> usize { + if x < y { y } else { x } +} + +#[cfg(test)] +// `InvalidRegexTest` intentionally contains an invalid regular expression. +#[allow(clippy::invalid_regex)] +mod tests { + use std::str::FromStr; + + use serde_json::{Number, Value, json}; + use stackable_operator::schemars::{JsonSchema, SchemaGenerator}; + use uuid::uuid; + + use super::{ErrorDiscriminants, Regex}; + use crate::framework::NameIsValidLabelValue; + + attributed_string_type! { + MinLengthWithoutConstraintsTest, + "min_length test without constraints", + "" + } + + #[test] + fn test_attributed_string_type_min_length_without_constraints() { + type T = MinLengthWithoutConstraintsTest; + + T::test_example(); + assert_eq!(0, T::MIN_LENGTH); + } + + attributed_string_type! { + MinLengthWithConstraintsTest, + "min_length test with constraints", + "test", + (min_length = 2), // should set the minimum length to 2 + (max_length = 8), // should not affect the minimum length + (regex = "^.{4}$"), // should not affect the minimum length + is_rfc_1035_label_name, // should be overruled by the greater min_length + is_valid_label_value // should be overruled by the greater min_length + } + + #[test] + fn test_attributed_string_type_min_length_with_constraints() { + type T = MinLengthWithConstraintsTest; + + T::test_example(); + assert_eq!(2, T::MIN_LENGTH); + assert_eq!( + Err(ErrorDiscriminants::MinimumLengthNotMet), + T::from_str("a").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + MaxLengthWithoutConstraintsTest, + "max_length test without constraints", + "" + } + + #[test] + fn test_attributed_string_type_max_length_without_constraints() { + type T = MaxLengthWithoutConstraintsTest; + + T::test_example(); + assert_eq!(usize::MAX, T::MAX_LENGTH); + } + + attributed_string_type! { + MaxLengthWithConstraintsTest, + "max_length test with constraints", + "test", + (min_length = 2), // should not affect the maximum length + (max_length = 8), // should set the maximum length to 8 + (regex = "^.{4}$"), // should not affect the maximum length + is_rfc_1035_label_name, // should be overruled by the lower max_length + is_valid_label_value // should be overruled by the lower max_length + } + + #[test] + fn test_attributed_string_type_max_length_with_constraints() { + type T = MaxLengthWithConstraintsTest; + + T::test_example(); + assert_eq!(8, T::MAX_LENGTH); + assert_eq!( + Err(ErrorDiscriminants::LengthExceeded), + T::from_str("test-12345").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + RegexWithoutConstraintsTest, + "regex test without constraints", + "" + } + + #[test] + fn test_attributed_string_type_regex_without_constraints() { + type T = RegexWithoutConstraintsTest; + + T::test_example(); + assert_eq!(Regex::MatchAll, T::REGEX); + } + + attributed_string_type! { + RegexWithOneConstraintTest, + "regex test with one constraint", + "test", + (min_length = 2), // should not affect the regular expression + (max_length = 8), // should not affect the regular expression + (regex = "^[est]{4}$") // should set the regular expression to "[est]{4}" + } + + #[test] + fn test_attributed_string_type_regex_with_one_constraint() { + type T = RegexWithOneConstraintTest; + + T::test_example(); + assert_eq!(Regex::Expression("^[est]{4}$"), T::REGEX); + assert_eq!( + Err(ErrorDiscriminants::RegexNotMatched), + T::from_str("t-st").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + RegexWithMultipleConstraintsTest, + "regex test with multiple constraints", + "test", + (min_length = 2), // should not affect the regular expression + (max_length = 8), // should not affect the regular expression + (regex = "^[est]{4}$"), // should not be combinable with is_rfc_1123_dns_subdomain_name + is_rfc_1123_dns_subdomain_name // should not be combinable with regex + } + + #[test] + fn test_attributed_string_type_regex_with_multiple_constraints() { + type T = RegexWithMultipleConstraintsTest; + + T::test_example(); + assert_eq!(Regex::Unknown, T::REGEX); + assert_eq!( + Err(ErrorDiscriminants::RegexNotMatched), + T::from_str("t-st").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + InvalidRegexTest, + "regex test with invalid expression", + "test", + (min_length = 2), // should not affect the regular expression + (max_length = 8), // should not affect the regular expression + (regex = "{") // should throw an error at runtime + } + + #[test] + fn test_attributed_string_type_regex_with_invalid_expression() { + type T = InvalidRegexTest; + + // It is not known yet at compile-time that this expression is invalid. + assert_eq!(Regex::Expression("{"), T::REGEX); + assert_eq!( + Err(ErrorDiscriminants::InvalidRegex), + T::from_str("test").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + DisplayFmtTest, + "Display::fmt test", + "test" + } + + #[test] + fn test_attributed_string_type_display_fmt() { + type T = DisplayFmtTest; + + assert_eq!("test", format!("{}", T::from_str_unsafe("test"))); + } + + attributed_string_type! { + StringFromTest, + "String::from test", + "test" + } + + #[test] + fn test_attributed_string_type_string_from() { + type T = StringFromTest; + + T::test_example(); + assert_eq!("test", String::from(T::from_str_unsafe("test"))); + assert_eq!("test", String::from(&T::from_str_unsafe("test"))); + } + + attributed_string_type! { + DeserializeTest, + "serde::Deserialize test", + "test", + (min_length = 2), + (max_length = 4), + (regex = "^[est-]+$"), + is_rfc_1035_label_name + } + + #[test] + fn test_attributed_string_type_deserialize() { + type T = DeserializeTest; + + T::test_example(); + assert_eq!( + T::from_str_unsafe("test"), + serde_json::from_value(Value::String("test".to_owned())) + .expect("should be deserializable") + ); + assert_eq!( + Err("minimum length not met".to_owned()), + serde_json::from_value::(Value::String("e".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("maximum length exceeded".to_owned()), + serde_json::from_value::(Value::String("testt".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("regular expression not matched".to_owned()), + serde_json::from_value::(Value::String("abc".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("not a valid label name as defined in RFC 1035".to_owned()), + serde_json::from_value::(Value::String("-tst".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: null, expected a string".to_owned()), + serde_json::from_value::(Value::Null).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: boolean `true`, expected a string".to_owned()), + serde_json::from_value::(Value::Bool(true)).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: integer `1`, expected a string".to_owned()), + serde_json::from_value::(Value::Number( + Number::from_i128(1).expect("should be a valid number") + )) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: sequence, expected a string".to_owned()), + serde_json::from_value::(Value::Array(vec![])).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: map, expected a string".to_owned()), + serde_json::from_value::(Value::Object(serde_json::Map::new())) + .map_err(|err| err.to_string()) + ); + } + + attributed_string_type! { + SerializeTest, + "serde::Serialize test", + "test" + } + + #[test] + fn test_attributed_string_type_serialize() { + type T = SerializeTest; + + T::test_example(); + assert_eq!( + "\"test\"".to_owned(), + serde_json::to_string(&T::from_str_unsafe("test")).expect("should be serializable") + ); + } + + attributed_string_type! { + JsonSchemaWithoutConstraintsTest, + "JsonSchema test with constraints", + "test" + } + + #[test] + fn test_attributed_string_type_json_schema_without_constaints() { + type T = JsonSchemaWithoutConstraintsTest; + + T::test_example(); + assert_eq!("JsonSchemaWithoutConstraintsTest", T::schema_name()); + assert_eq!( + json!({ + "type": "string", + "minLength": 0, + "maxLength": None::, + "pattern": None:: + }), + T::json_schema(&mut SchemaGenerator::default()) + ); + } + + attributed_string_type! { + JsonSchemaWithConstraintsTest, + "JsonSchema test with constraints", + "test", + (min_length = 4), + (max_length = 8), + (regex = "^[est]+$") + } + + #[test] + fn test_attributed_string_type_json_schema_with_constraints() { + type T = JsonSchemaWithConstraintsTest; + + T::test_example(); + assert_eq!("JsonSchemaWithConstraintsTest", T::schema_name()); + assert_eq!( + json!({ + "type": "string", + "minLength": 4, + "maxLength": 8, + "pattern": "^[est]+$" + }), + T::json_schema(&mut SchemaGenerator::default()) + ); + } + + attributed_string_type! { + IsRfc1035LabelNameTest, + "is_rfc_1035_label_name test", + "a-b", + is_rfc_1035_label_name + } + + #[test] + fn test_attributed_string_type_is_rfc_1035_label_name() { + type T = IsRfc1035LabelNameTest; + + let _ = T::IS_RFC_1035_LABEL_NAME; + let _ = T::IS_RFC_1123_LABEL_NAME; + let _ = T::IS_RFC_1123_SUBDOMAIN_NAME; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidRfc1035LabelName), + T::from_str("A").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + IsRfc1123DnsSubdomainNameTest, + "is_rfc_1123_dns_subdomain_name test", + "a-b.c", + is_rfc_1123_dns_subdomain_name + } + + #[test] + fn test_attributed_string_type_is_rfc_1123_dns_subdomain_name() { + type T = IsRfc1123DnsSubdomainNameTest; + + let _ = T::IS_RFC_1123_SUBDOMAIN_NAME; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidRfc1123DnsSubdomainName), + T::from_str("A").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + IsRfc1123LabelNameTest, + "is_rfc_1123_label_name test", + "1-a", + is_rfc_1123_label_name + } + + #[test] + fn test_attributed_string_type_is_rfc_1123_label_name() { + type T = IsRfc1123LabelNameTest; + + let _ = T::IS_RFC_1123_LABEL_NAME; + let _ = T::IS_RFC_1123_SUBDOMAIN_NAME; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidRfc1123LabelName), + T::from_str("A").map_err(ErrorDiscriminants::from) + ); + } + + attributed_string_type! { + IsValidLabelValueTest, + "is_valid_label_value test", + "a-_.1", + is_valid_label_value + } + + #[test] + fn test_attributed_string_type_is_valid_label_value() { + type T = IsValidLabelValueTest; + + let _ = T::IS_VALID_LABEL_VALUE; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidLabelValue), + T::from_str("invalid label value").map_err(ErrorDiscriminants::from) + ); + assert_eq!( + "label-value", + T::from_str_unsafe("label-value").to_label_value() + ); + } + + attributed_string_type! { + IsUidTest, + "is_uid test", + "c27b3971-ca72-42c1-80a4-abdfc1db0ddd", + is_uid + } + + #[test] + fn test_attributed_string_type_is_uid() { + type T = IsUidTest; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidUid), + T::from_str("invalid UID").map_err(ErrorDiscriminants::from) + ); + assert_eq!( + "c27b3971-ca72-42c1-80a4-abdfc1db0ddd", + T::from(uuid!("c27b3971-ca72-42c1-80a4-abdfc1db0ddd")).to_string() + ); + assert_eq!( + "c27b3971-ca72-42c1-80a4-abdfc1db0ddd", + T::from(&uuid!("c27b3971-ca72-42c1-80a4-abdfc1db0ddd")).to_string() + ); + } +} diff --git a/rust/operator-binary/src/framework/macros/constant.rs b/rust/operator-binary/src/framework/macros/constant.rs new file mode 100644 index 00000000..ae4e9c69 --- /dev/null +++ b/rust/operator-binary/src/framework/macros/constant.rs @@ -0,0 +1,17 @@ +/// Use [`std::sync::LazyLock`] to define a static "constant" from a string. +/// +/// The string is converted into the given type with [`std::str::FromStr::from_str`]. +/// +/// # Examples +/// +/// ```rust +/// constant!(DATA_VOLUME_NAME: VolumeName = "data"); +/// constant!(pub CONFIG_VOLUME_NAME: VolumeName = "config"); +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! constant { + ($qualifier:vis $name:ident: $type:ident = $value:literal) => { + $qualifier static $name: std::sync::LazyLock<$type> = + std::sync::LazyLock::new(|| $type::from_str($value).expect("should be a valid $name")); + }; +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 6cc89e99..eb2ebc1c 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -43,6 +43,7 @@ mod config; mod controller_commons; mod crd; mod env_vars; +mod framework; mod operations; mod product_logging; mod service; From b5991c5c87d2cc70060c4f486fc67f3a62ec1588 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:29:46 +0200 Subject: [PATCH 03/14] feat: add framework validated types for names and resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce strongly-typed wrappers for Kubernetes and operator names: - types/common: Port type with range validation - types/kubernetes: ConfigMapName, ServiceName, StatefulSetName, ListenerName, NamespaceName, Uid — each with appropriate RFC constraints and max_length limits - types/operator: ProductName, ClusterName, RoleName, RoleGroupName, ControllerName, OperatorName — with operator-specific length budgets ensuring the combined -- fits within StatefulSet name limits (52 chars) Operator-specific values (max_length, examples) are flagged with REVIEW comments for parameterisation when this module moves to operator-rs. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/framework.rs | 1 + rust/operator-binary/src/framework/types.rs | 3 + .../src/framework/types/common.rs | 68 +++++++ .../src/framework/types/kubernetes.rs | 191 ++++++++++++++++++ .../src/framework/types/operator.rs | 108 ++++++++++ 5 files changed, 371 insertions(+) create mode 100644 rust/operator-binary/src/framework/types.rs create mode 100644 rust/operator-binary/src/framework/types/common.rs create mode 100644 rust/operator-binary/src/framework/types/kubernetes.rs create mode 100644 rust/operator-binary/src/framework/types/operator.rs diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 0115ccc3..830bbcca 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -21,6 +21,7 @@ #[macro_use] pub mod macros; +pub mod types; /// The name is a valid label value pub trait NameIsValidLabelValue { diff --git a/rust/operator-binary/src/framework/types.rs b/rust/operator-binary/src/framework/types.rs new file mode 100644 index 00000000..65f61166 --- /dev/null +++ b/rust/operator-binary/src/framework/types.rs @@ -0,0 +1,3 @@ +pub mod common; +pub mod kubernetes; +pub mod operator; diff --git a/rust/operator-binary/src/framework/types/common.rs b/rust/operator-binary/src/framework/types/common.rs new file mode 100644 index 00000000..3d7326ef --- /dev/null +++ b/rust/operator-binary/src/framework/types/common.rs @@ -0,0 +1,68 @@ +//! Common types that do not belong (yet) to a more specific module +use snafu::{ResultExt, Snafu}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to convert to port number"))] + ConvertToPortNumber { source: std::num::TryFromIntError }, +} + +/// A port number +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Port(pub u16); + +impl std::fmt::Display for Port { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl From for Port { + fn from(value: u16) -> Self { + Port(value) + } +} + +impl From for i32 { + fn from(value: Port) -> Self { + value.0 as i32 + } +} + +impl TryFrom for Port { + type Error = Error; + + fn try_from(value: i32) -> Result { + Ok(Port( + u16::try_from(value).context(ConvertToPortNumberSnafu)?, + )) + } +} + +#[cfg(test)] +mod tests { + + use super::{ErrorDiscriminants, Port}; + + #[test] + fn test_port_fmt() { + assert_eq!("0".to_owned(), Port(0).to_string()); + assert_eq!("65535".to_owned(), Port(65535).to_string()); + } + + #[test] + fn test_port_try_from_i32() { + assert_eq!(Some(Port(0)), Port::try_from(0).ok()); + assert_eq!(Some(Port(65535)), Port::try_from(65535).ok()); + assert_eq!( + Err(ErrorDiscriminants::ConvertToPortNumber), + Port::try_from(-1).map_err(ErrorDiscriminants::from) + ); + assert_eq!( + Err(ErrorDiscriminants::ConvertToPortNumber), + Port::try_from(65536).map_err(ErrorDiscriminants::from) + ); + } +} diff --git a/rust/operator-binary/src/framework/types/kubernetes.rs b/rust/operator-binary/src/framework/types/kubernetes.rs new file mode 100644 index 00000000..3902e5a0 --- /dev/null +++ b/rust/operator-binary/src/framework/types/kubernetes.rs @@ -0,0 +1,191 @@ +//! Kubernetes (resource) names +use std::str::FromStr; + +use stackable_operator::validation::{RFC_1123_LABEL_MAX_LENGTH, RFC_1123_SUBDOMAIN_MAX_LENGTH}; + +use crate::attributed_string_type; + +attributed_string_type! { + ConfigMapName, + "The name of a ConfigMap", + "airflow-webserver-default", + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + ConfigMapKey, + "The key for a ConfigMap", + "webserver_config.py", + (min_length = 1), + // see https://github.com/kubernetes/kubernetes/blob/v1.34.1/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L435-L451 + (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), + (regex = "^[-._a-zA-Z0-9]+$") +} + +attributed_string_type! { + ContainerName, + "The name of a container in a Pod", + "airflow", + is_rfc_1123_label_name +} + +attributed_string_type! { + ClusterRoleName, + "The name of a ClusterRole", + "airflow-clusterrole", + // On the one hand, ClusterRoles must only contain characters that are allowed for DNS + // subdomain names, on the other hand, their length does not seem to be restricted – at least + // on Kind. However, 253 characters are sufficient for the Stackable operators, and to avoid + // problems on other Kubernetes providers, the length is restricted here. + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + Hostname, + "A hostname", + "example.com", + (min_length = 1), + (max_length = 253), + // see https://en.wikipedia.org/wiki/Hostname#Syntax + (regex = "^[a-zA-Z0-9]([-a-zA-Z0-9]{0,60}[a-zA-Z0-9])?(\\.[a-zA-Z0-9]([-a-zA-Z0-9]{0,60}[a-zA-Z0-9])?)*\\.?$") +} + +attributed_string_type! { + ListenerName, + "The name of a Listener", + "airflow-webserver-default", + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + ListenerClassName, + "The name of a ListenerClass", + "external-stable", + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + NamespaceName, + "The name of a Namespace", + "stackable-operators", + is_rfc_1123_label_name, + is_valid_label_value +} + +attributed_string_type! { + PersistentVolumeClaimName, + "The name of a PersistentVolumeClaim", + "config", + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + RoleBindingName, + "The name of a RoleBinding", + "airflow-rolebinding", + // On the one hand, RoleBindings must only contain characters that are allowed for DNS + // subdomain names, on the other hand, their length does not seem to be restricted – at least + // on Kind. However, 253 characters are sufficient for the Stackable operators, and to avoid + // problems on other Kubernetes providers, the length is restricted here. + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + SecretClassName, + "The name of a SecretClass", + "tls", + // The secret class name is used in an annotation on the tls volume. + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + SecretKey, + "The key for a Secret", + "accessKey", + (min_length = 1), + // see https://github.com/kubernetes/kubernetes/blob/v1.34.1/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L435-L451 + (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), + (regex = "^[-._a-zA-Z0-9]+$") +} + +attributed_string_type! { + SecretName, + "The name of a Secret", + "airflow-internal-secret", + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + ServiceAccountName, + "The name of a ServiceAccount", + "airflow-serviceaccount", + is_rfc_1123_dns_subdomain_name +} + +attributed_string_type! { + ServiceName, + "The name of a Service", + "airflow-webserver-default", + is_rfc_1035_label_name, + is_valid_label_value +} + +attributed_string_type! { + StatefulSetName, + "The name of a StatefulSet", + "airflow-webserver-default", + (max_length = + // see https://github.com/kubernetes/kubernetes/issues/64023 + RFC_1123_LABEL_MAX_LENGTH + - 1 /* dash */ + - 10 /* digits for the controller-revision-hash label */), + is_rfc_1123_label_name, + is_valid_label_value +} + +attributed_string_type! { + Uid, + "A UID", + "c27b3971-ca72-42c1-80a4-abdfc1db0ddd", + is_uid, + is_valid_label_value +} + +attributed_string_type! { + VolumeName, + "The name of a Volume", + "config", + is_rfc_1123_label_name, + is_valid_label_value +} + +#[cfg(test)] +mod tests { + use super::{ + ClusterRoleName, ConfigMapKey, ConfigMapName, ContainerName, Hostname, ListenerClassName, + ListenerName, NamespaceName, PersistentVolumeClaimName, RoleBindingName, SecretClassName, + SecretKey, SecretName, ServiceAccountName, ServiceName, StatefulSetName, Uid, VolumeName, + }; + + #[test] + fn test_attributed_string_type_examples() { + ConfigMapName::test_example(); + ConfigMapKey::test_example(); + ContainerName::test_example(); + ClusterRoleName::test_example(); + Hostname::test_example(); + ListenerName::test_example(); + ListenerClassName::test_example(); + NamespaceName::test_example(); + PersistentVolumeClaimName::test_example(); + RoleBindingName::test_example(); + SecretClassName::test_example(); + SecretKey::test_example(); + SecretName::test_example(); + ServiceAccountName::test_example(); + ServiceName::test_example(); + StatefulSetName::test_example(); + Uid::test_example(); + VolumeName::test_example(); + } +} diff --git a/rust/operator-binary/src/framework/types/operator.rs b/rust/operator-binary/src/framework/types/operator.rs new file mode 100644 index 00000000..8e4d3f50 --- /dev/null +++ b/rust/operator-binary/src/framework/types/operator.rs @@ -0,0 +1,108 @@ +//! Names for operators +//! +//! Several types below have operator-specific max_length values and examples that are +//! hardcoded for airflow. When this module moves to operator-rs, these should be +//! parameterised so each operator can supply its own limits. The compile-time assertions +//! in role_group_utils.rs verify that the limits are consistent with each other. + +use std::str::FromStr; + +use crate::attributed_string_type; + +attributed_string_type! { + ProductName, + "The name of a product", + // REVIEW: example is operator-specific — parameterise when moving to operator-rs + "airflow", + // A suffix is added to produce a label value. An according compile-time check ensures that + // max_length cannot be set higher. + (max_length = 54), + is_rfc_1123_dns_subdomain_name, + is_valid_label_value +} + +attributed_string_type! { + ProductVersion, + "The version of a product", + "2.10.4", + is_valid_label_value +} + +attributed_string_type! { + ClusterName, + "The name of a cluster/stacklet", + // REVIEW: example is operator-specific — parameterise when moving to operator-rs + "my-airflow-cluster", + // Suffixes are added to produce resource names. According compile-time checks ensure that + // max_length cannot be set higher. Reduced from opensearch's 24 to 22 because airflow's + // longest role name ("dagprocessor") is 12 chars vs opensearch's 10. + // REVIEW: max_length is operator-specific (depends on longest RoleName) — parameterise + // when moving to operator-rs + (max_length = 22), + is_rfc_1035_label_name, + is_valid_label_value +} + +attributed_string_type! { + ControllerName, + "The name of a controller in an operator", + // REVIEW: example is operator-specific — parameterise when moving to operator-rs + "airflowcluster", + is_valid_label_value +} + +attributed_string_type! { + OperatorName, + "The name of an operator", + // REVIEW: example is operator-specific — parameterise when moving to operator-rs + "airflow.stackable.tech", + is_valid_label_value +} + +attributed_string_type! { + RoleGroupName, + "The name of a role-group name", + "default", + // The role-group name is used to produce resource names. To make sure that all resource names + // are valid, max_length is restricted. Compile-time checks ensure that max_length cannot be + // set higher if not other names like the RoleName are set lower accordingly. + // REVIEW: max_length is operator-specific (depends on ClusterName and RoleName budgets) — + // parameterise when moving to operator-rs + (max_length = 16), + is_rfc_1123_label_name, + is_valid_label_value +} + +attributed_string_type! { + RoleName, + "The name of a role name", + // REVIEW: example is operator-specific — parameterise when moving to operator-rs + "webserver", + // The role name is used to produce resource names. To make sure that all resource names are + // valid, max_length is restricted. Compile-time checks ensure that max_length cannot be set + // higher if not other names like the RoleGroupName are set lower accordingly. + // REVIEW: max_length is operator-specific (airflow needs 12 for "dagprocessor", opensearch + // uses 10) — parameterise when moving to operator-rs + (max_length = 12), + is_rfc_1123_label_name, + is_valid_label_value +} + +#[cfg(test)] +mod tests { + use super::{ + ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, RoleGroupName, + RoleName, + }; + + #[test] + fn test_attributed_string_type_examples() { + ProductName::test_example(); + ProductVersion::test_example(); + ClusterName::test_example(); + ControllerName::test_example(); + OperatorName::test_example(); + RoleGroupName::test_example(); + RoleName::test_example(); + } +} From 635bac3eb577993ccf21409af9c565b2d52b3c4d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:31:04 +0200 Subject: [PATCH 04/14] feat: add framework KVP module for type-safe labels Add label construction functions that accept validated types (ProductName, OperatorName, ClusterName, etc.) instead of raw strings, eliminating the possibility of label value constraint violations at the call site. Provides: recommended_labels, role_group_selector_labels, prometheus_labels, and prometheus_annotations. The HasName trait is added to framework.rs as it is required by the label functions to extract names from cluster objects. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/framework.rs | 10 + rust/operator-binary/src/framework/kvp.rs | 1 + .../src/framework/kvp/label.rs | 207 ++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 rust/operator-binary/src/framework/kvp.rs create mode 100644 rust/operator-binary/src/framework/kvp/label.rs diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 830bbcca..cf8b8ddb 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -21,8 +21,18 @@ #[macro_use] pub mod macros; +pub mod kvp; pub mod types; +/// Has a non-empty name +/// +/// Useful as an object reference; Should not be used to create an object because the name could +/// violate the naming constraints (e.g. maximum length) of the object. +pub trait HasName { + #[allow(dead_code)] + fn to_name(&self) -> String; +} + /// The name is a valid label value pub trait NameIsValidLabelValue { fn to_label_value(&self) -> String; diff --git a/rust/operator-binary/src/framework/kvp.rs b/rust/operator-binary/src/framework/kvp.rs new file mode 100644 index 00000000..0006163a --- /dev/null +++ b/rust/operator-binary/src/framework/kvp.rs @@ -0,0 +1 @@ +pub mod label; diff --git a/rust/operator-binary/src/framework/kvp/label.rs b/rust/operator-binary/src/framework/kvp/label.rs new file mode 100644 index 00000000..101e239e --- /dev/null +++ b/rust/operator-binary/src/framework/kvp/label.rs @@ -0,0 +1,207 @@ +use stackable_operator::{ + kube::Resource, + kvp::{Labels, ObjectLabels}, +}; + +use crate::framework::{ + HasName, NameIsValidLabelValue, + types::operator::{ + ControllerName, OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, + }, +}; + +/// Infallible variant of [`stackable_operator::kvp::Labels::recommended`] +pub fn recommended_labels( + owner: &(impl Resource + HasName + NameIsValidLabelValue), + product_name: &ProductName, + product_version: &ProductVersion, + operator_name: &OperatorName, + controller_name: &ControllerName, + role_name: &RoleName, + role_group_name: &RoleGroupName, +) -> Labels { + let object_labels = ObjectLabels { + owner, + app_name: &product_name.to_label_value(), + app_version: &product_version.to_label_value(), + operator_name: &operator_name.to_label_value(), + controller_name: &controller_name.to_label_value(), + role: &role_name.to_label_value(), + role_group: &role_group_name.to_label_value(), + }; + Labels::recommended(&object_labels).expect( + "Labels should be created because the owner has an object name and all given parameters \ + produce valid label values.", + ) +} + +/// Infallible variant of [`stackable_operator::kvp::Labels::role_selector`] +#[allow(dead_code)] +pub fn role_selector( + owner: &(impl Resource + HasName + NameIsValidLabelValue), + product_name: &ProductName, + role_name: &RoleName, +) -> Labels { + Labels::role_selector( + owner, + &product_name.to_label_value(), + &role_name.to_label_value(), + ) + .expect("Labels should be created because all given parameters produce valid label values") +} + +/// Infallible variant of [`stackable_operator::kvp::Labels::role_group_selector`] +pub fn role_group_selector( + owner: &(impl Resource + HasName + NameIsValidLabelValue), + product_name: &ProductName, + role_name: &RoleName, + role_group_name: &RoleGroupName, +) -> Labels { + Labels::role_group_selector( + owner, + &product_name.to_label_value(), + &role_name.to_label_value(), + &role_group_name.to_label_value(), + ) + .expect("Labels should be created because all given parameters produce valid label values") +} + +#[cfg(test)] +mod tests { + use std::{borrow::Cow, collections::BTreeMap}; + + use stackable_operator::{ + k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, kube::Resource, + }; + + use crate::framework::{ + HasName, NameIsValidLabelValue, + kvp::label::{recommended_labels, role_group_selector, role_selector}, + types::operator::{ + ControllerName, OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, + }, + }; + + struct Cluster { + object_meta: ObjectMeta, + } + + impl Cluster { + fn new() -> Self { + Cluster { + object_meta: ObjectMeta { + name: Some("cluster-name".to_owned()), + ..ObjectMeta::default() + }, + } + } + } + + impl Resource for Cluster { + type DynamicType = (); + type Scope = (); + + fn kind(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("AirflowCluster") + } + + fn group(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("airflow.stackable.tech") + } + + fn version(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("v1alpha2") + } + + fn plural(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("airflowclusters") + } + + fn meta(&self) -> &ObjectMeta { + &self.object_meta + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.object_meta + } + } + + impl HasName for Cluster { + fn to_name(&self) -> String { + self.object_meta.name.clone().expect("set in new()") + } + } + + impl NameIsValidLabelValue for Cluster { + fn to_label_value(&self) -> String { + self.object_meta.name.clone().expect("set in new()") + } + } + + #[test] + fn test_recommended_labels() { + let actual_labels = recommended_labels( + &Cluster::new(), + &ProductName::from_str_unsafe("my-product"), + &ProductVersion::from_str_unsafe("1.0.0"), + &OperatorName::from_str_unsafe("my-operator"), + &ControllerName::from_str_unsafe("my-controller"), + &RoleName::from_str_unsafe("my-role"), + &RoleGroupName::from_str_unsafe("my-role-group"), + ); + + let expected_labels: BTreeMap = [ + ("app.kubernetes.io/component", "my-role"), + ("app.kubernetes.io/instance", "cluster-name"), + ("app.kubernetes.io/managed-by", "my-operator_my-controller"), + ("app.kubernetes.io/name", "my-product"), + ("app.kubernetes.io/role-group", "my-role-group"), + ("app.kubernetes.io/version", "1.0.0"), + ("stackable.tech/vendor", "Stackable"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(); + + assert_eq!(expected_labels, actual_labels.into()); + } + + #[test] + fn test_role_selector() { + let actual_labels = role_selector( + &Cluster::new(), + &ProductName::from_str_unsafe("my-product"), + &RoleName::from_str_unsafe("my-role"), + ); + + let expected_labels: BTreeMap = [ + ("app.kubernetes.io/component", "my-role"), + ("app.kubernetes.io/instance", "cluster-name"), + ("app.kubernetes.io/name", "my-product"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(); + + assert_eq!(expected_labels, actual_labels.into()); + } + + #[test] + fn test_role_group_selector() { + let actual_labels = role_group_selector( + &Cluster::new(), + &ProductName::from_str_unsafe("my-product"), + &RoleName::from_str_unsafe("my-role"), + &RoleGroupName::from_str_unsafe("my-role-group"), + ); + + let expected_labels: BTreeMap = [ + ("app.kubernetes.io/component", "my-role"), + ("app.kubernetes.io/instance", "cluster-name"), + ("app.kubernetes.io/name", "my-product"), + ("app.kubernetes.io/role-group", "my-role-group"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(); + + assert_eq!(expected_labels, actual_labels.into()); + } +} From 4b068645f1611a28df9f4a095ccdce98b5c7ac63 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:32:17 +0200 Subject: [PATCH 05/14] feat: add framework builders for K8s resources Add type-safe builder functions for constructing Kubernetes resources using validated types instead of raw strings: - builder/meta: ObjectMeta with validated names, labels, owner refs - builder/pdb: PodDisruptionBudget with role-aware max_unavailable - builder/pod/container: container construction with ports, env vars, volume mounts, resource limits, probes - builder/pod/volume: config map and empty dir volume helpers - builder/statefulset: StatefulSet with pod template, service name, replicas, restart policy The HasUid trait is added to framework.rs as it is required by the meta builder for constructing owner references. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/framework.rs | 8 + rust/operator-binary/src/framework/builder.rs | 6 + .../src/framework/builder/meta.rs | 108 ++++++ .../src/framework/builder/pdb.rs | 177 +++++++++ .../src/framework/builder/pod.rs | 2 + .../src/framework/builder/pod/container.rs | 367 ++++++++++++++++++ .../src/framework/builder/pod/volume.rs | 48 +++ .../src/framework/builder/statefulset.rs | 118 ++++++ 8 files changed, 834 insertions(+) create mode 100644 rust/operator-binary/src/framework/builder.rs create mode 100644 rust/operator-binary/src/framework/builder/meta.rs create mode 100644 rust/operator-binary/src/framework/builder/pdb.rs create mode 100644 rust/operator-binary/src/framework/builder/pod.rs create mode 100644 rust/operator-binary/src/framework/builder/pod/container.rs create mode 100644 rust/operator-binary/src/framework/builder/pod/volume.rs create mode 100644 rust/operator-binary/src/framework/builder/statefulset.rs diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index cf8b8ddb..563fa234 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -21,9 +21,12 @@ #[macro_use] pub mod macros; +pub mod builder; pub mod kvp; pub mod types; +use types::kubernetes::Uid; + /// Has a non-empty name /// /// Useful as an object reference; Should not be used to create an object because the name could @@ -33,6 +36,11 @@ pub trait HasName { fn to_name(&self) -> String; } +/// Has a Kubernetes UID +pub trait HasUid { + fn to_uid(&self) -> Uid; +} + /// The name is a valid label value pub trait NameIsValidLabelValue { fn to_label_value(&self) -> String; diff --git a/rust/operator-binary/src/framework/builder.rs b/rust/operator-binary/src/framework/builder.rs new file mode 100644 index 00000000..5d02a0b0 --- /dev/null +++ b/rust/operator-binary/src/framework/builder.rs @@ -0,0 +1,6 @@ +pub mod meta; +pub mod pdb; +#[allow(dead_code)] +pub mod pod; +#[allow(dead_code)] +pub mod statefulset; diff --git a/rust/operator-binary/src/framework/builder/meta.rs b/rust/operator-binary/src/framework/builder/meta.rs new file mode 100644 index 00000000..7ebb3cc7 --- /dev/null +++ b/rust/operator-binary/src/framework/builder/meta.rs @@ -0,0 +1,108 @@ +use stackable_operator::{ + builder::meta::OwnerReferenceBuilder, + k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference, kube::Resource, +}; + +use crate::framework::{HasName, HasUid}; + +/// Infallible variant of +/// [`stackable_operator::builder::meta::ObjectMetaBuilder::ownerreference_from_resource`] +pub fn ownerreference_from_resource( + resource: &(impl Resource + HasName + HasUid), + block_owner_deletion: Option, + controller: Option, +) -> OwnerReference { + OwnerReferenceBuilder::new() + // Set api_version, kind, name and additionally the UID if it exists. + .initialize_from_resource(resource) + // Ensure that the name is set. + .name(resource.to_name()) + // Ensure that the UID is set. + .uid(resource.to_uid().to_string()) + .block_owner_deletion_opt(block_owner_deletion) + .controller_opt(controller) + .build() + .expect( + "OwnerReference should be created because the resource has an api_version, kind, name \ + and uid.", + ) +} + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use stackable_operator::{ + k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, kube::Resource, + }; + + use crate::framework::{ + HasName, HasUid, builder::meta::ownerreference_from_resource, types::kubernetes::Uid, + }; + + struct TestCluster { + object_meta: ObjectMeta, + } + + impl TestCluster { + fn new() -> Self { + TestCluster { + object_meta: ObjectMeta { + name: Some("test-cluster".to_owned()), + uid: Some("a6b89911-d48e-4328-88d6-b9251226583d".to_owned()), + ..ObjectMeta::default() + }, + } + } + } + + impl Resource for TestCluster { + type DynamicType = (); + type Scope = (); + + fn kind(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("AirflowCluster") + } + + fn group(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("airflow.stackable.tech") + } + + fn version(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("v1alpha2") + } + + fn plural(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("airflowclusters") + } + + fn meta(&self) -> &ObjectMeta { + &self.object_meta + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.object_meta + } + } + + impl HasName for TestCluster { + fn to_name(&self) -> String { + self.object_meta.name.clone().expect("set in new()") + } + } + + impl HasUid for TestCluster { + fn to_uid(&self) -> Uid { + Uid::from_str_unsafe(&self.object_meta.uid.clone().expect("set in new()")) + } + } + + #[test] + fn test_ownerreference_from_resource() { + let owner_ref = ownerreference_from_resource(&TestCluster::new(), Some(true), Some(true)); + assert_eq!(owner_ref.name, "test-cluster"); + assert_eq!(owner_ref.uid, "a6b89911-d48e-4328-88d6-b9251226583d"); + assert_eq!(owner_ref.controller, Some(true)); + assert_eq!(owner_ref.block_owner_deletion, Some(true)); + } +} diff --git a/rust/operator-binary/src/framework/builder/pdb.rs b/rust/operator-binary/src/framework/builder/pdb.rs new file mode 100644 index 00000000..9cf22af8 --- /dev/null +++ b/rust/operator-binary/src/framework/builder/pdb.rs @@ -0,0 +1,177 @@ +use stackable_operator::{ + builder::pdb::PodDisruptionBudgetBuilder, + k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector, + kube::{Resource, api::ObjectMeta}, +}; + +use crate::framework::{ + HasName, HasUid, NameIsValidLabelValue, + types::operator::{ControllerName, OperatorName, ProductName, RoleName}, +}; + +/// Infallible variant of +/// [`stackable_operator::builder::pdb::PodDisruptionBudgetBuilder::new_with_role`] +pub fn pod_disruption_budget_builder_with_role( + owner: &(impl Resource + HasName + NameIsValidLabelValue + HasUid), + product_name: &ProductName, + role_name: &RoleName, + operator_name: &OperatorName, + controller_name: &ControllerName, +) -> PodDisruptionBudgetBuilder { + PodDisruptionBudgetBuilder::new_with_role( + owner, + &product_name.to_label_value(), + &role_name.to_label_value(), + &operator_name.to_label_value(), + &controller_name.to_label_value(), + ) + .expect( + "PodDisruptionBudgetBuilder should be created because the owner has an object name and UID \ + and all given parameters produce valid label values.", + ) +} + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use stackable_operator::{ + k8s_openapi::{ + api::policy::v1::{PodDisruptionBudget, PodDisruptionBudgetSpec}, + apimachinery::pkg::{ + apis::meta::v1::{LabelSelector, ObjectMeta, OwnerReference}, + util::intstr::IntOrString, + }, + }, + kube::Resource, + }; + + use crate::framework::{ + HasName, HasUid, NameIsValidLabelValue, + builder::pdb::pod_disruption_budget_builder_with_role, + types::{ + kubernetes::Uid, + operator::{ControllerName, OperatorName, ProductName, RoleName}, + }, + }; + + struct Cluster { + object_meta: ObjectMeta, + } + + impl Cluster { + fn new() -> Self { + Cluster { + object_meta: ObjectMeta { + name: Some("cluster-name".to_owned()), + uid: Some("a6b89911-d48e-4328-88d6-b9251226583d".to_owned()), + ..ObjectMeta::default() + }, + } + } + } + + impl Resource for Cluster { + type DynamicType = (); + type Scope = (); + + fn kind(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("AirflowCluster") + } + + fn group(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("airflow.stackable.tech") + } + + fn version(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("v1alpha2") + } + + fn plural(_dt: &Self::DynamicType) -> Cow<'_, str> { + Cow::from("airflowclusters") + } + + fn meta(&self) -> &ObjectMeta { + &self.object_meta + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.object_meta + } + } + + impl HasName for Cluster { + fn to_name(&self) -> String { + self.object_meta.name.clone().expect("set in new()") + } + } + + impl HasUid for Cluster { + fn to_uid(&self) -> Uid { + Uid::from_str_unsafe(&self.object_meta.uid.clone().expect("set in new()")) + } + } + + impl NameIsValidLabelValue for Cluster { + fn to_label_value(&self) -> String { + self.object_meta.name.clone().expect("set in new()") + } + } + + #[test] + fn test_pod_disruption_budget_builder_with_role() { + let actual_pdb = pod_disruption_budget_builder_with_role( + &Cluster::new(), + &ProductName::from_str_unsafe("my-product"), + &RoleName::from_str_unsafe("my-role"), + &OperatorName::from_str_unsafe("my-operator"), + &ControllerName::from_str_unsafe("my-controller"), + ) + .with_max_unavailable(2) + .build(); + + let expected_pdb = PodDisruptionBudget { + metadata: ObjectMeta { + labels: Some( + [ + ("app.kubernetes.io/component", "my-role"), + ("app.kubernetes.io/instance", "cluster-name"), + ("app.kubernetes.io/managed-by", "my-operator_my-controller"), + ("app.kubernetes.io/name", "my-product"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(), + ), + name: Some("cluster-name-my-role".to_owned()), + owner_references: Some(vec![OwnerReference { + api_version: "airflow.stackable.tech/v1alpha2".to_owned(), + controller: Some(true), + kind: "AirflowCluster".to_owned(), + name: "cluster-name".to_owned(), + uid: "a6b89911-d48e-4328-88d6-b9251226583d".to_owned(), + ..OwnerReference::default() + }]), + ..ObjectMeta::default() + }, + spec: Some(PodDisruptionBudgetSpec { + max_unavailable: Some(IntOrString::Int(2)), + selector: Some(LabelSelector { + match_labels: Some( + [ + ("app.kubernetes.io/component", "my-role"), + ("app.kubernetes.io/instance", "cluster-name"), + ("app.kubernetes.io/name", "my-product"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(), + ), + ..LabelSelector::default() + }), + ..PodDisruptionBudgetSpec::default() + }), + ..PodDisruptionBudget::default() + }; + + assert_eq!(expected_pdb, actual_pdb); + } +} diff --git a/rust/operator-binary/src/framework/builder/pod.rs b/rust/operator-binary/src/framework/builder/pod.rs new file mode 100644 index 00000000..df93bd44 --- /dev/null +++ b/rust/operator-binary/src/framework/builder/pod.rs @@ -0,0 +1,2 @@ +pub mod container; +pub mod volume; diff --git a/rust/operator-binary/src/framework/builder/pod/container.rs b/rust/operator-binary/src/framework/builder/pod/container.rs new file mode 100644 index 00000000..244bf003 --- /dev/null +++ b/rust/operator-binary/src/framework/builder/pod/container.rs @@ -0,0 +1,367 @@ +use std::{collections::BTreeMap, fmt::Display, str::FromStr}; + +use snafu::Snafu; +use stackable_operator::{ + builder::pod::container::{ContainerBuilder, FieldPathEnvVar}, + k8s_openapi::api::core::v1::{ConfigMapKeySelector, EnvVar, EnvVarSource, ObjectFieldSelector}, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::framework::types::kubernetes::{ConfigMapKey, ConfigMapName, ContainerName}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display( + "invalid environment variable name: a valid environment variable name must not be empty \ + and must consist only of printable ASCII characters other than '='" + ))] + ParseEnvVarName { env_var_name: String }, +} + +/// Infallible variant of [`stackable_operator::builder::pod::container::ContainerBuilder::new`] +pub fn new_container_builder(container_name: &ContainerName) -> ContainerBuilder { + ContainerBuilder::new(container_name.as_ref()).expect("should be a valid container name") +} + +// TODO Use attributed_string_type instead +/// Validated environment variable name +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct EnvVarName(String); + +impl EnvVarName { + /// Creates an [`EnvVarName`] from the given string and panics if the validation failed + /// + /// Use this only with constant names that are also tested in unit tests! + pub fn from_str_unsafe(s: &str) -> Self { + EnvVarName::from_str(s).expect("should be a valid environment variable name") + } +} + +impl Display for EnvVarName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl FromStr for EnvVarName { + type Err = Error; + + fn from_str(s: &str) -> Result { + // The length of environment variable names seems not to be restricted. + + if !s.is_empty() && s.chars().all(|c| matches!(c, ' '..='<' | '>'..='~')) { + Ok(Self(s.to_owned())) + } else { + Err(Error::ParseEnvVarName { + env_var_name: s.to_owned(), + }) + } + } +} + +/// A set of [`EnvVar`]s +/// +/// The environment variable names in the set are unique. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct EnvVarSet(BTreeMap); + +impl EnvVarSet { + /// Creates an empty [`EnvVarSet`] + pub fn new() -> Self { + Self::default() + } + + /// Returns a reference to the [`EnvVar`] with the given name + pub fn get(&self, env_var_name: &EnvVarName) -> Option<&EnvVar> { + self.0.get(env_var_name) + } + + /// Moves all [`EnvVar`]s from the given set into this one. + /// + /// [`EnvVar`]s with the same name are overridden. + pub fn merge(mut self, mut env_var_set: EnvVarSet) -> Self { + self.0.append(&mut env_var_set.0); + + self + } + + /// Adds the given [`EnvVar`]s to this set + /// + /// [`EnvVar`]s with the same name are overridden. + pub fn with_values(self, env_vars: I) -> Self + where + I: IntoIterator, + V: Into, + { + env_vars + .into_iter() + .fold(self, |extended_env_vars, (name, value)| { + extended_env_vars.with_value(&name, value) + }) + } + + /// Adds an environment variable with the given name and string value to this set + /// + /// An [`EnvVar`] with the same name is overridden. + pub fn with_value(mut self, name: &EnvVarName, value: impl Into) -> Self { + self.0.insert( + name.clone(), + EnvVar { + name: name.to_string(), + value: Some(value.into()), + value_from: None, + }, + ); + + self + } + + /// Adds an environment variable with the given name and field path to this set + /// + /// An [`EnvVar`] with the same name is overridden. + pub fn with_field_path(mut self, name: &EnvVarName, field_path: FieldPathEnvVar) -> Self { + self.0.insert( + name.clone(), + EnvVar { + name: name.to_string(), + value: None, + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: field_path.to_string(), + ..ObjectFieldSelector::default() + }), + ..EnvVarSource::default() + }), + }, + ); + + self + } + + /// Adds an environment variable with the given ConfigMap key reference to this set + /// + /// An [`EnvVar`] with the same name is overridden. + pub fn with_config_map_key_ref( + mut self, + name: &EnvVarName, + config_map_name: &ConfigMapName, + config_map_key: &ConfigMapKey, + ) -> Self { + self.0.insert( + name.clone(), + EnvVar { + name: name.to_string(), + value: None, + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + key: config_map_key.to_string(), + name: config_map_name.to_string(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + }, + ); + + self + } +} + +impl From for Vec { + fn from(value: EnvVarSet) -> Self { + value.0.values().cloned().collect() + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + builder::pod::container::FieldPathEnvVar, + k8s_openapi::api::core::v1::{ + ConfigMapKeySelector, EnvVar, EnvVarSource, ObjectFieldSelector, + }, + }; + + use super::{EnvVarName, EnvVarSet}; + use crate::framework::{ + builder::pod::container::new_container_builder, + types::kubernetes::{ConfigMapKey, ConfigMapName, ContainerName}, + }; + + #[test] + fn test_envvarname_fromstr() { + // actually accepted by Kubernetes + assert!(EnvVarName::from_str(" !\"#$%&'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~").is_ok()); + + // empty string + assert!(EnvVarName::from_str("").is_err()); + // non-printable ASCII characters + assert!(EnvVarName::from_str("\n").is_err()); + assert!(EnvVarName::from_str("€").is_err()); + // equals sign + assert!(EnvVarName::from_str("=").is_err()); + } + + #[test] + fn test_new_container_builder() { + // Test that the function does not panic + new_container_builder(&ContainerName::from_str_unsafe("valid-container-name")); + } + + #[test] + fn test_envvarname_format() { + assert_eq!( + "TEST".to_owned(), + format!("{}", EnvVarName::from_str_unsafe("TEST")) + ); + } + + #[test] + fn test_envvarset_merge() { + let env_var_set1 = EnvVarSet::new().with_values([ + ( + EnvVarName::from_str_unsafe("ENV1"), + "value1 from env_var_set1", + ), + ( + EnvVarName::from_str_unsafe("ENV2"), + "value2 from env_var_set1", + ), + ( + EnvVarName::from_str_unsafe("ENV3"), + "value3 from env_var_set1", + ), + ]); + let env_var_set2 = EnvVarSet::new() + .with_value( + &EnvVarName::from_str_unsafe("ENV2"), + "value2 from env_var_set2", + ) + .with_field_path(&EnvVarName::from_str_unsafe("ENV3"), FieldPathEnvVar::Name) + .with_value( + &EnvVarName::from_str_unsafe("ENV4"), + "value4 from env_var_set2", + ); + + let merged_env_var_set = env_var_set1.merge(env_var_set2); + + assert_eq!( + vec![ + EnvVar { + name: "ENV1".to_owned(), + value: Some("value1 from env_var_set1".to_owned()), + value_from: None + }, + EnvVar { + name: "ENV2".to_owned(), + value: Some("value2 from env_var_set2".to_owned()), + value_from: None + }, + EnvVar { + name: "ENV3".to_owned(), + value: None, + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: "metadata.name".to_owned(), + ..ObjectFieldSelector::default() + }), + ..EnvVarSource::default() + }), + }, + EnvVar { + name: "ENV4".to_owned(), + value: Some("value4 from env_var_set2".to_owned()), + value_from: None + } + ], + Vec::from(merged_env_var_set) + ); + } + + #[test] + fn test_envvarset_with_values() { + let env_var_set = EnvVarSet::new().with_values([ + (EnvVarName::from_str_unsafe("ENV1"), "value1"), + (EnvVarName::from_str_unsafe("ENV2"), "value2"), + ]); + + assert_eq!( + vec![ + EnvVar { + name: "ENV1".to_owned(), + value: Some("value1".to_owned()), + value_from: None + }, + EnvVar { + name: "ENV2".to_owned(), + value: Some("value2".to_owned()), + value_from: None + } + ], + Vec::from(env_var_set) + ); + } + + #[test] + fn test_envvarset_with_value() { + let env_var_set = EnvVarSet::new().with_value(&EnvVarName::from_str_unsafe("ENV"), "value"); + + assert_eq!( + Some(&EnvVar { + name: "ENV".to_owned(), + value: Some("value".to_owned()), + value_from: None + }), + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) + ); + } + + #[test] + fn test_envvarset_with_field_path() { + let env_var_set = EnvVarSet::new() + .with_field_path(&EnvVarName::from_str_unsafe("ENV"), FieldPathEnvVar::Name); + + assert_eq!( + Some(&EnvVar { + name: "ENV".to_owned(), + value: None, + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: "metadata.name".to_owned(), + ..ObjectFieldSelector::default() + }), + ..EnvVarSource::default() + }), + }), + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) + ); + } + + #[test] + fn test_envvarset_with_config_map_key_ref() { + let env_var_set = EnvVarSet::new().with_config_map_key_ref( + &EnvVarName::from_str_unsafe("ENV"), + &ConfigMapName::from_str_unsafe("config-map"), + &ConfigMapKey::from_str_unsafe("key"), + ); + + assert_eq!( + Some(&EnvVar { + name: "ENV".to_owned(), + value: None, + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + key: "key".to_owned(), + name: "config-map".to_owned(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + }), + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) + ); + } +} diff --git a/rust/operator-binary/src/framework/builder/pod/volume.rs b/rust/operator-binary/src/framework/builder/pod/volume.rs new file mode 100644 index 00000000..06dc4846 --- /dev/null +++ b/rust/operator-binary/src/framework/builder/pod/volume.rs @@ -0,0 +1,48 @@ +use stackable_operator::{ + builder::pod::volume::ListenerOperatorVolumeSourceBuilder, + k8s_openapi::api::core::v1::PersistentVolumeClaim, kvp::Labels, +}; + +use crate::framework::types::kubernetes::{ + ListenerClassName, ListenerName, PersistentVolumeClaimName, +}; + +/// Infallible variant of [`stackable_operator::builder::pod::volume::ListenerReference`] +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ListenerReference { + ListenerClass(ListenerClassName), + Listener(ListenerName), +} + +impl From<&ListenerReference> for stackable_operator::builder::pod::volume::ListenerReference { + fn from(value: &ListenerReference) -> Self { + match value { + ListenerReference::ListenerClass(listener_class_name) => { + stackable_operator::builder::pod::volume::ListenerReference::ListenerClass( + listener_class_name.to_string(), + ) + } + ListenerReference::Listener(listener_name) => { + stackable_operator::builder::pod::volume::ListenerReference::ListenerName( + listener_name.to_string(), + ) + } + } + } +} + +/// Infallible variant of +/// [`stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilder::build_pvc`] +pub fn listener_operator_volume_source_builder_build_pvc( + listener_reference: &ListenerReference, + labels: &Labels, + pvc_name: &PersistentVolumeClaimName, +) -> PersistentVolumeClaim { + ListenerOperatorVolumeSourceBuilder::new(&listener_reference.into(), labels) + .build_pvc(pvc_name.to_string()) + .expect( + "should return a PersistentVolumeClaim, because the only check is that \ + listener_reference is a valid annotation value and there are no restrictions on single \ + annotation values", + ) +} diff --git a/rust/operator-binary/src/framework/builder/statefulset.rs b/rust/operator-binary/src/framework/builder/statefulset.rs new file mode 100644 index 00000000..904d333b --- /dev/null +++ b/rust/operator-binary/src/framework/builder/statefulset.rs @@ -0,0 +1,118 @@ +use std::collections::BTreeMap; + +use stackable_operator::kvp::Annotations; + +use crate::framework::types::kubernetes::{ConfigMapName, SecretName}; + +/// Creates `restarter.stackable.tech/ignore-configmap.{i}` annotations for each given ConfigMap. +/// +/// The restarter uses these annotations to skip restarting Pods when specific ConfigMaps change. +/// Indices start at 0 and are assigned in iteration order, so **do not merge the result with +/// annotations from another call** — duplicate indices would overwrite each other. +pub fn restarter_ignore_configmap_annotations( + ignored_config_maps: impl IntoIterator, +) -> Annotations { + let annotation_key_values = ignored_config_maps + .into_iter() + .enumerate() + .map(|(i, config_map_name)| { + ( + format!("restarter.stackable.tech/ignore-configmap.{i}"), + config_map_name.to_string(), + ) + }) + .collect::>(); + + Annotations::try_from(annotation_key_values).expect( + "should contain only valid annotations because the annotation keys are statically \ + defined apart from the index number and the names of ConfigMaps are valid annotation \ + values.", + ) +} + +/// Creates `restarter.stackable.tech/ignore-secret.{i}` annotations for each given Secret. +/// +/// The restarter uses these annotations to skip restarting Pods when specific Secrets change. +/// Indices start at 0 and are assigned in iteration order, so **do not merge the result with +/// annotations from another call** — duplicate indices would overwrite each other. +pub fn restarter_ignore_secret_annotations( + ignored_secrets: impl IntoIterator, +) -> Annotations { + let annotation_key_values = ignored_secrets + .into_iter() + .enumerate() + .map(|(i, secret_name)| { + ( + format!("restarter.stackable.tech/ignore-secret.{i}"), + secret_name.to_string(), + ) + }) + .collect::>(); + + Annotations::try_from(annotation_key_values).expect( + "should contain only valid annotations because the annotation keys are statically \ + defined apart from the index number and the names of Secrets are valid annotation \ + values.", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn multiple_config_maps_produce_indexed_annotations() { + let ignored_config_maps = [ + ConfigMapName::from_str_unsafe("first-config"), + ConfigMapName::from_str_unsafe("second-config"), + ConfigMapName::from_str_unsafe("third-config"), + ]; + + let actual_annotations = restarter_ignore_configmap_annotations(ignored_config_maps); + + let expected_annotations = BTreeMap::from([ + ( + "restarter.stackable.tech/ignore-configmap.0".to_owned(), + "first-config".to_owned(), + ), + ( + "restarter.stackable.tech/ignore-configmap.1".to_owned(), + "second-config".to_owned(), + ), + ( + "restarter.stackable.tech/ignore-configmap.2".to_owned(), + "third-config".to_owned(), + ), + ]); + + assert_eq!(expected_annotations, actual_annotations.into()); + } + + #[test] + fn multiple_secrets_produce_indexed_annotations() { + let ignored_secrets = [ + SecretName::from_str_unsafe("first-secret"), + SecretName::from_str_unsafe("second-secret"), + SecretName::from_str_unsafe("third-secret"), + ]; + + let actual_annotations = restarter_ignore_secret_annotations(ignored_secrets); + + let expected_annotations = BTreeMap::from([ + ( + "restarter.stackable.tech/ignore-secret.0".to_owned(), + "first-secret".to_owned(), + ), + ( + "restarter.stackable.tech/ignore-secret.1".to_owned(), + "second-secret".to_owned(), + ), + ( + "restarter.stackable.tech/ignore-secret.2".to_owned(), + "third-secret".to_owned(), + ), + ]); + + assert_eq!(expected_annotations, actual_annotations.into()); + } +} From 2443c6fc858104ed37b1dfd227f38ebb4639dd6e Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:33:29 +0200 Subject: [PATCH 06/14] feat: add framework utilities for roles, resources, and logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the framework module with: - role_utils: role config validation, merging, and iteration with validated types — replaces stringly-typed role/rolegroup handling - role_group_utils: QualifiedRoleGroupName with compile-time length assertions ensuring -- fits within StatefulSet, Service, ConfigMap, and Listener name limits - controller_utils: config merging for role/rolegroup hierarchies, producing validated config per rolegroup - cluster_resources: typed wrapper around operator-rs ClusterResources - product_logging: validated logging config types (ValidatedContainerLogConfigChoice, VectorContainerLogConfig) Modules marked #[allow(dead_code)] are not yet referenced by the controller — they will be wired in when the controller pipeline is replaced in a later commit. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/framework.rs | 9 + .../src/framework/cluster_resources.rs | 50 +++ .../src/framework/controller_utils.rs | 211 ++++++++++ .../src/framework/product_logging.rs | 1 + .../framework/product_logging/framework.rs | 127 ++++++ .../src/framework/role_group_utils.rs | 151 +++++++ .../src/framework/role_utils.rs | 386 ++++++++++++++++++ 7 files changed, 935 insertions(+) create mode 100644 rust/operator-binary/src/framework/cluster_resources.rs create mode 100644 rust/operator-binary/src/framework/controller_utils.rs create mode 100644 rust/operator-binary/src/framework/product_logging.rs create mode 100644 rust/operator-binary/src/framework/product_logging/framework.rs create mode 100644 rust/operator-binary/src/framework/role_group_utils.rs create mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 563fa234..cacec8e0 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -22,7 +22,16 @@ #[macro_use] pub mod macros; pub mod builder; +#[allow(dead_code)] +pub mod cluster_resources; +#[allow(dead_code)] +pub mod controller_utils; pub mod kvp; +pub mod product_logging; +#[allow(dead_code)] +pub mod role_group_utils; +#[allow(dead_code)] +pub mod role_utils; pub mod types; use types::kubernetes::Uid; diff --git a/rust/operator-binary/src/framework/cluster_resources.rs b/rust/operator-binary/src/framework/cluster_resources.rs new file mode 100644 index 00000000..430b534f --- /dev/null +++ b/rust/operator-binary/src/framework/cluster_resources.rs @@ -0,0 +1,50 @@ +use stackable_operator::{ + cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, + deep_merger::ObjectOverrides, + k8s_openapi::api::core::v1::ObjectReference, +}; + +use super::types::{ + kubernetes::{NamespaceName, Uid}, + operator::{ClusterName, ControllerName, OperatorName, ProductName}, +}; +use crate::framework::{ + NameIsValidLabelValue, macros::attributed_string_type::MAX_LABEL_VALUE_LENGTH, +}; + +/// Infallible variant of [`stackable_operator::cluster_resources::ClusterResources::new`] +#[allow(clippy::too_many_arguments)] +pub fn cluster_resources_new<'a>( + product_name: &ProductName, + operator_name: &OperatorName, + controller_name: &ControllerName, + cluster_name: &ClusterName, + cluster_namespace: &NamespaceName, + cluster_uid: &Uid, + apply_strategy: ClusterResourceApplyStrategy, + object_overrides: &'a ObjectOverrides, +) -> ClusterResources<'a> { + // compile-time check + // ClusterResources::new creates a label value from the given app name by appending + // `-operator`. For the resulting label value to be valid, it must not exceed + // MAX_LABEL_VALUE_LENGTH. + const _: () = assert!( + ProductName::MAX_LENGTH + "-operator".len() <= MAX_LABEL_VALUE_LENGTH, + "The string `-operator` must not exceed the limit of Label names." + ); + + ClusterResources::new( + &product_name.to_label_value(), + &operator_name.to_label_value(), + &controller_name.to_label_value(), + &ObjectReference { + name: Some(cluster_name.to_string()), + namespace: Some(cluster_namespace.to_string()), + uid: Some(cluster_uid.to_string()), + ..Default::default() + }, + apply_strategy, + object_overrides, + ) + .expect("ClusterResources should be created because the cluster object reference contains name, namespace and uid.") +} diff --git a/rust/operator-binary/src/framework/controller_utils.rs b/rust/operator-binary/src/framework/controller_utils.rs new file mode 100644 index 00000000..d15e53f5 --- /dev/null +++ b/rust/operator-binary/src/framework/controller_utils.rs @@ -0,0 +1,211 @@ +//! Helper functions which are not tied to a specific controller step + +use std::str::FromStr; + +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::kube::runtime::reflector::Lookup; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::framework::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, +}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to get the cluster name"))] + GetClusterName {}, + + #[snafu(display("failed to get the namespace"))] + GetNamespace {}, + + #[snafu(display("failed to get the UID"))] + GetUid {}, + + #[snafu(display("failed to set the cluster name"))] + ParseClusterName { + source: crate::framework::macros::attributed_string_type::Error, + }, + + #[snafu(display("failed to set the namespace"))] + ParseNamespace { + source: crate::framework::macros::attributed_string_type::Error, + }, + + #[snafu(display("failed to set the UID"))] + ParseUid { + source: crate::framework::macros::attributed_string_type::Error, + }, +} + +type Result = std::result::Result; + +/// Get the cluster name from the given resource +pub fn get_cluster_name(cluster: &impl Lookup) -> Result { + let raw_cluster_name = cluster.name().context(GetClusterNameSnafu)?; + let cluster_name = ClusterName::from_str(&raw_cluster_name).context(ParseClusterNameSnafu)?; + + Ok(cluster_name) +} + +/// Get the namespace from the given resource +pub fn get_namespace(resource: &impl Lookup) -> Result { + let raw_namespace = resource.namespace().context(GetNamespaceSnafu)?; + let namespace = NamespaceName::from_str(&raw_namespace).context(ParseNamespaceSnafu)?; + + Ok(namespace) +} + +/// Get the UID from the given resource +pub fn get_uid(resource: &impl Lookup) -> Result { + let raw_uid = resource.uid().context(GetUidSnafu)?; + let uid = Uid::from_str(&raw_uid).context(ParseUidSnafu)?; + + Ok(uid) +} + +#[cfg(test)] +mod tests { + use stackable_operator::kube::runtime::reflector::Lookup; + use uuid::uuid; + + use super::{ErrorDiscriminants, get_cluster_name, get_namespace, get_uid}; + use crate::framework::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }; + + #[derive(Debug, Default)] + struct TestResource { + name: Option<&'static str>, + namespace: Option<&'static str>, + uid: Option<&'static str>, + } + + impl Lookup for TestResource { + type DynamicType = (); + + fn kind(_dyntype: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + "TestResource".into() + } + + fn group(_dyntype: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + "stackable.tech".into() + } + + fn version(_dyntype: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + "v1".into() + } + + fn plural(_dyntype: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + "testresources".into() + } + + fn name(&self) -> Option> { + self.name.map(std::borrow::Cow::Borrowed) + } + + fn namespace(&self) -> Option> { + self.namespace.map(std::borrow::Cow::Borrowed) + } + + fn resource_version(&self) -> Option> { + Some("1".into()) + } + + fn uid(&self) -> Option> { + self.uid.map(std::borrow::Cow::Borrowed) + } + } + + #[test] + fn test_get_cluster_name() { + assert_eq!( + ClusterName::from_str_unsafe("test-cluster"), + get_cluster_name(&TestResource { + name: Some("test-cluster"), + ..TestResource::default() + }) + .expect("should contain a valid cluster name") + ); + + assert_eq!( + Err(ErrorDiscriminants::GetClusterName), + get_cluster_name(&TestResource { + name: None, + ..TestResource::default() + }) + .map_err(ErrorDiscriminants::from) + ); + + assert_eq!( + Err(ErrorDiscriminants::ParseClusterName), + get_cluster_name(&TestResource { + name: Some("invalid cluster name"), + ..TestResource::default() + }) + .map_err(ErrorDiscriminants::from) + ); + } + + #[test] + fn test_get_namespace() { + assert_eq!( + NamespaceName::from_str_unsafe("test-namespace"), + get_namespace(&TestResource { + namespace: Some("test-namespace"), + ..TestResource::default() + }) + .expect("should contain a valid namespace") + ); + + assert_eq!( + Err(ErrorDiscriminants::GetNamespace), + get_namespace(&TestResource { + namespace: None, + ..TestResource::default() + }) + .map_err(ErrorDiscriminants::from) + ); + + assert_eq!( + Err(ErrorDiscriminants::ParseNamespace), + get_namespace(&TestResource { + namespace: Some("invalid namespace"), + ..TestResource::default() + }) + .map_err(ErrorDiscriminants::from) + ); + } + + #[test] + fn test_get_uid() { + assert_eq!( + Uid::from(uuid!("e6ac237d-a6d4-43a1-8135-f36506110912")), + get_uid(&TestResource { + uid: Some("e6ac237d-a6d4-43a1-8135-f36506110912"), + ..TestResource::default() + }) + .expect("should contain a valid UID") + ); + + assert_eq!( + Err(ErrorDiscriminants::GetUid), + get_uid(&TestResource { + uid: None, + ..TestResource::default() + }) + .map_err(ErrorDiscriminants::from) + ); + + assert_eq!( + Err(ErrorDiscriminants::ParseUid), + get_uid(&TestResource { + uid: Some("invalid UID"), + ..TestResource::default() + }) + .map_err(ErrorDiscriminants::from) + ); + } +} diff --git a/rust/operator-binary/src/framework/product_logging.rs b/rust/operator-binary/src/framework/product_logging.rs new file mode 100644 index 00000000..0c717499 --- /dev/null +++ b/rust/operator-binary/src/framework/product_logging.rs @@ -0,0 +1 @@ +pub mod framework; diff --git a/rust/operator-binary/src/framework/product_logging/framework.rs b/rust/operator-binary/src/framework/product_logging/framework.rs new file mode 100644 index 00000000..76a5c04b --- /dev/null +++ b/rust/operator-binary/src/framework/product_logging/framework.rs @@ -0,0 +1,127 @@ +use std::fmt::Display; + +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::product_logging::spec::{ + AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, Logging, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::framework::types::kubernetes::ConfigMapName; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to get container log configuration for container {container}"))] + GetContainerLogConfiguration { container: String }, + + #[snafu(display("failed to parse ConfigMap name for custom log configuration"))] + ParseConfigMapName { + source: crate::framework::macros::attributed_string_type::Error, + }, +} + +#[derive(Clone, Debug)] +pub enum ValidatedContainerLogConfigChoice { + Automatic(AutomaticContainerLogConfig), + Custom(ConfigMapName), +} + +impl ValidatedContainerLogConfigChoice { + /// Converts back to the raw upstream type for use at API boundaries + /// (e.g. calling `product_logging::framework::vector_container`). + pub fn to_raw_container_log_config(&self) -> ContainerLogConfig { + match self { + Self::Automatic(auto) => ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(auto.clone())), + }, + Self::Custom(name) => ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { + config_map: name.to_string(), + }, + })), + }, + } + } +} + +#[derive(Clone, Debug)] +pub struct VectorContainerLogConfig { + pub log_config: ValidatedContainerLogConfigChoice, +} + +pub fn validate_logging_configuration_for_container( + logging: &Logging, + container: T, +) -> Result +where + T: Clone + Display + Ord, +{ + use std::str::FromStr; + + let config = logging + .containers + .get(&container) + .and_then(|c| c.choice.as_ref()) + .context(GetContainerLogConfigurationSnafu { + container: container.to_string(), + })?; + + match config { + ContainerLogConfigChoice::Automatic(automatic) => Ok( + ValidatedContainerLogConfigChoice::Automatic(automatic.clone()), + ), + ContainerLogConfigChoice::Custom(custom) => { + let config_map_name = ConfigMapName::from_str(&custom.custom.config_map) + .context(ParseConfigMapNameSnafu)?; + Ok(ValidatedContainerLogConfigChoice::Custom(config_map_name)) + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use stackable_operator::product_logging::spec::{ + AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging, + }; + + use super::*; + use crate::crd::Container; + + fn logging_with_automatic_config() -> Logging { + let mut containers = BTreeMap::new(); + containers.insert( + Container::Airflow, + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + ); + Logging { + enable_vector_agent: false, + containers, + } + } + + #[test] + fn test_validate_automatic_log_config() { + let logging = logging_with_automatic_config(); + let result = validate_logging_configuration_for_container(&logging, Container::Airflow); + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + ValidatedContainerLogConfigChoice::Automatic(_) + )); + } + + #[test] + fn test_validate_missing_container_config() { + let logging = logging_with_automatic_config(); + let result = validate_logging_configuration_for_container(&logging, Container::Vector); + assert!(result.is_err()); + } +} diff --git a/rust/operator-binary/src/framework/role_group_utils.rs b/rust/operator-binary/src/framework/role_group_utils.rs new file mode 100644 index 00000000..61ea8637 --- /dev/null +++ b/rust/operator-binary/src/framework/role_group_utils.rs @@ -0,0 +1,151 @@ +use std::str::FromStr; + +use super::types::{ + kubernetes::{ConfigMapName, ListenerName, ServiceName, StatefulSetName}, + operator::{ClusterName, RoleGroupName, RoleName}, +}; +use crate::attributed_string_type; + +attributed_string_type! { + QualifiedRoleGroupName, + "A qualified role group name consisting of the cluster name, role name and role-group name. It is a valid label name as defined in RFC 1035 that can be used e.g. as a name for a Service or a StatefulSet.", + "airflow-webserver-default", + // Suffixes are added to produce resource names. According compile-time checks ensure that + // max_length cannot be set higher. + (max_length = 52), + is_rfc_1035_label_name, + is_valid_label_value +} + +/// Type-safe names for role-group resources +pub struct ResourceNames { + pub cluster_name: ClusterName, + pub role_name: RoleName, + pub role_group_name: RoleGroupName, +} + +impl ResourceNames { + /// Creates a qualified role group name in the format + /// `--` + fn qualified_role_group_name(&self) -> QualifiedRoleGroupName { + // compile-time checks + const _: () = assert!( + ClusterName::MAX_LENGTH + + 1 // dash + + RoleName::MAX_LENGTH + + 1 // dash + + RoleGroupName::MAX_LENGTH + <= QualifiedRoleGroupName::MAX_LENGTH, + "The string `--` must not exceed the limit \ + of RFC 1035 label names." + ); + // qualified_role_group_name is only an RFC 1035 label name if it starts with an + // alphabetic character, therefore cluster_name must also be an RFC 1035 label name. + // role_name and role_group_name and the middle of the qualified_role_group_name can + // be RFC 1123 label names because digits are allowed there. + let _ = ClusterName::IS_RFC_1035_LABEL_NAME; + let _ = RoleName::IS_RFC_1123_LABEL_NAME; + let _ = RoleGroupName::IS_RFC_1123_LABEL_NAME; + + QualifiedRoleGroupName::from_str(&format!( + "{}-{}-{}", + self.cluster_name, self.role_name, self.role_group_name, + )) + .expect("should be a valid QualifiedRoleGroupName") + } + + pub fn role_group_config_map(&self) -> ConfigMapName { + // compile-time check + const _: () = assert!( + QualifiedRoleGroupName::MAX_LENGTH <= ConfigMapName::MAX_LENGTH, + "The string `--` must not exceed the limit of \ + ConfigMap names." + ); + let _ = QualifiedRoleGroupName::IS_RFC_1123_SUBDOMAIN_NAME; + + ConfigMapName::from_str(self.qualified_role_group_name().as_ref()) + .expect("should be a valid ConfigMap name") + } + + pub fn stateful_set_name(&self) -> StatefulSetName { + // compile-time checks + const _: () = assert!( + QualifiedRoleGroupName::MAX_LENGTH <= StatefulSetName::MAX_LENGTH, + "The string `--` must not exceed the \ + limit of StatefulSet names." + ); + let _ = QualifiedRoleGroupName::IS_RFC_1123_LABEL_NAME; + let _ = QualifiedRoleGroupName::IS_VALID_LABEL_VALUE; + + StatefulSetName::from_str(self.qualified_role_group_name().as_ref()) + .expect("should be a valid StatefulSet name") + } + + pub fn headless_service_name(&self) -> ServiceName { + const SUFFIX: &str = "-headless"; + + const _: () = assert!( + QualifiedRoleGroupName::MAX_LENGTH + SUFFIX.len() <= ServiceName::MAX_LENGTH, + "The string `---headless` must not exceed the \ + limit of Service names." + ); + let _ = QualifiedRoleGroupName::IS_RFC_1035_LABEL_NAME; + let _ = QualifiedRoleGroupName::IS_VALID_LABEL_VALUE; + + ServiceName::from_str(&format!("{}{SUFFIX}", self.qualified_role_group_name())) + .expect("should be a valid Service name") + } + + pub fn listener_name(&self) -> ListenerName { + const _: () = assert!( + QualifiedRoleGroupName::MAX_LENGTH <= ListenerName::MAX_LENGTH, + "The string `--` must not exceed the limit of \ + Listener names." + ); + let _ = QualifiedRoleGroupName::IS_RFC_1123_SUBDOMAIN_NAME; + + ListenerName::from_str(self.qualified_role_group_name().as_ref()) + .expect("should be a valid Listener name") + } +} + +#[cfg(test)] +mod tests { + use super::{ClusterName, RoleGroupName, RoleName}; + use crate::framework::{ + role_group_utils::{QualifiedRoleGroupName, ResourceNames}, + types::kubernetes::{ConfigMapName, ListenerName, ServiceName, StatefulSetName}, + }; + + #[test] + fn test_resource_names() { + QualifiedRoleGroupName::test_example(); + + let resource_names = ResourceNames { + cluster_name: ClusterName::from_str_unsafe("test-cluster"), + role_name: RoleName::from_str_unsafe("webserver"), + role_group_name: RoleGroupName::from_str_unsafe("default"), + }; + + assert_eq!( + QualifiedRoleGroupName::from_str_unsafe("test-cluster-webserver-default"), + resource_names.qualified_role_group_name() + ); + assert_eq!( + ConfigMapName::from_str_unsafe("test-cluster-webserver-default"), + resource_names.role_group_config_map() + ); + assert_eq!( + StatefulSetName::from_str_unsafe("test-cluster-webserver-default"), + resource_names.stateful_set_name() + ); + assert_eq!( + ServiceName::from_str_unsafe("test-cluster-webserver-default-headless"), + resource_names.headless_service_name() + ); + assert_eq!( + ListenerName::from_str_unsafe("test-cluster-webserver-default"), + resource_names.listener_name() + ); + } +} diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs new file mode 100644 index 00000000..de61ac54 --- /dev/null +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -0,0 +1,386 @@ +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, +}; + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + config::{ + fragment::{self, FromFragment}, + merge::{Merge, merge}, + }, + k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, + role_utils::{CommonConfiguration, Role, RoleGroup}, + schemars::{self, JsonSchema}, +}; + +use super::{ + builder::pod::container::EnvVarSet, + types::{ + kubernetes::{ClusterRoleName, RoleBindingName, ServiceAccountName}, + operator::{ClusterName, ProductName}, + }, +}; + +/// Variant of `stackable_operator::role_utils::GenericCommonConfig` that +/// implements [`Merge`] +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] +pub struct GenericProductSpecificCommonConfig {} + +impl Merge for GenericProductSpecificCommonConfig { + fn merge(&mut self, _defaults: &Self) {} +} + +/// Variant of [`stackable_operator::role_utils::RoleGroup`] that is easier to work with +/// +/// Differences are: +/// * `replicas` is non-optional. +/// * `config` is flattened. +/// * The [`HashMap`] in `env_overrides` is replaced with an [`EnvVarSet`]. +#[derive(Clone, Debug, PartialEq)] +pub struct RoleGroupConfig { + pub replicas: u16, + pub config: T, + pub config_overrides: HashMap>, + pub env_overrides: EnvVarSet, + pub cli_overrides: BTreeMap, + pub pod_overrides: PodTemplateSpec, + // allow(dead_code) is not necessary anymore when moved to operator-rs + #[allow(dead_code)] + pub product_specific_common_config: ProductSpecificCommonConfig, +} + +impl RoleGroupConfig { + pub fn cli_overrides_to_vec(&self) -> Vec { + self.cli_overrides + .clone() + .into_iter() + .flat_map(|(option, value)| [option, value]) + .collect() + } +} + +/// Variant of [`stackable_operator::role_utils::RoleGroup::validate_config`] with fixed types +/// +/// The `role` parameter takes the `ProductSpecificCommonConfig` into account. +pub fn validate_config( + role_group: &RoleGroup, + role: &Role, + default_config: &T, +) -> Result +where + C: FromFragment, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, + T: Merge + Clone, + RoleConfig: Default + JsonSchema + Serialize, +{ + let mut role_config = role.config.config.clone(); + role_config.merge(default_config); + let mut rolegroup_config = role_group.config.config.clone(); + rolegroup_config.merge(&role_config); + fragment::validate(rolegroup_config) +} + +/// Merges and validates the [`RoleGroup`] with the given `role` and `default_config` +pub fn with_validated_config( + role_group: &RoleGroup, + role: &Role, + default_config: &T, +) -> Result, fragment::ValidationError> +where + C: FromFragment, + CommonConfig: Clone + Default + JsonSchema + Merge + Serialize, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, + T: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, +{ + let validated_config = validate_config(role_group, role, default_config)?; + Ok(RoleGroup { + config: CommonConfiguration { + config: validated_config, + config_overrides: merge( + role_group.config.config_overrides.clone(), + &role.config.config_overrides, + ), + env_overrides: merged_env_overrides( + role.config.env_overrides.clone(), + role_group.config.env_overrides.clone(), + ), + cli_overrides: merged_cli_overrides( + role.config.cli_overrides.clone(), + role_group.config.cli_overrides.clone(), + ), + pod_overrides: merged_pod_overrides( + role.config.pod_overrides.clone(), + role_group.config.pod_overrides.clone(), + ), + product_specific_common_config: merge( + role_group.config.product_specific_common_config.clone(), + &role.config.product_specific_common_config, + ), + }, + replicas: role_group.replicas, + }) +} + +fn merged_env_overrides( + role_env_overrides: HashMap, + role_group_env_overrides: HashMap, +) -> HashMap { + let mut merged_env_overrides = role_env_overrides; + merged_env_overrides.extend(role_group_env_overrides); + merged_env_overrides +} + +fn merged_cli_overrides( + role_cli_overrides: BTreeMap, + role_group_cli_overrides: BTreeMap, +) -> BTreeMap { + let mut merged_cli_overrides = role_cli_overrides; + merged_cli_overrides.extend(role_group_cli_overrides); + merged_cli_overrides +} + +fn merged_pod_overrides( + role_pod_overrides: PodTemplateSpec, + role_group_pod_overrides: PodTemplateSpec, +) -> PodTemplateSpec { + let mut merged_pod_overrides = role_pod_overrides; + merged_pod_overrides.merge_from(role_group_pod_overrides); + merged_pod_overrides +} + +/// Type-safe names for role resources +pub struct ResourceNames { + pub cluster_name: ClusterName, + pub product_name: ProductName, +} + +impl ResourceNames { + pub fn service_account_name(&self) -> ServiceAccountName { + const SUFFIX: &str = "-serviceaccount"; + + // compile-time checks + const _: () = assert!( + ClusterName::MAX_LENGTH + SUFFIX.len() <= ServiceAccountName::MAX_LENGTH, + "The string `-serviceaccount` must not exceed the limit of ServiceAccount names." + ); + let _ = ClusterName::IS_RFC_1123_SUBDOMAIN_NAME; + + ServiceAccountName::from_str(&format!("{}{SUFFIX}", self.cluster_name)) + .expect("should be a valid ServiceAccount name") + } + + pub fn role_binding_name(&self) -> RoleBindingName { + const SUFFIX: &str = "-rolebinding"; + + // compile-time checks + const _: () = assert!( + ClusterName::MAX_LENGTH + SUFFIX.len() <= RoleBindingName::MAX_LENGTH, + "The string `-rolebinding` must not exceed the limit of RoleBinding names." + ); + let _ = ClusterName::IS_RFC_1123_SUBDOMAIN_NAME; + + RoleBindingName::from_str(&format!("{}{SUFFIX}", self.cluster_name)) + .expect("should be a valid RoleBinding name") + } + + pub fn cluster_role_name(&self) -> ClusterRoleName { + const SUFFIX: &str = "-clusterrole"; + + // compile-time checks + const _: () = assert!( + ProductName::MAX_LENGTH + SUFFIX.len() <= ClusterRoleName::MAX_LENGTH, + "The string `-clusterrole` must not exceed the limit of cluster role names." + ); + let _ = ProductName::IS_RFC_1123_SUBDOMAIN_NAME; + + ClusterRoleName::from_str(&format!("{}{SUFFIX}", self.product_name)) + .expect("should be a valid cluster role name") + } +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, HashMap}; + + use rstest::*; + use serde::Serialize; + use stackable_operator::{ + config::{fragment::Fragment, merge::Merge}, + k8s_openapi::api::core::v1::PodTemplateSpec, + kube::api::ObjectMeta, + role_utils::{CommonConfiguration, GenericRoleConfig, Role, RoleGroup}, + schemars::{self, JsonSchema}, + }; + + use super::ResourceNames; + use crate::framework::{ + role_utils::with_validated_config, + types::{ + kubernetes::{ClusterRoleName, RoleBindingName, ServiceAccountName}, + operator::{ClusterName, ProductName}, + }, + }; + + #[derive(Debug, Fragment, PartialEq)] + #[fragment_attrs(derive(Clone, Debug, Default, Merge, PartialEq))] + struct Config { + property: String, + } + + impl Config { + fn new(value: &str) -> Self { + Self { + property: value.to_owned(), + } + } + } + + impl ConfigFragment { + fn new(value: Option<&str>) -> Self { + Self { + property: value.map(str::to_owned), + } + } + } + + #[derive(Clone, Debug, Default, JsonSchema, Merge, PartialEq, Serialize)] + struct ProductCommonConfig { + property: Option, + } + + #[derive(Clone, Debug, Default, JsonSchema, Merge, PartialEq, Serialize)] + struct TestConfigOverrides { + property: Option, + } + + fn new_common_config( + config: T, + override_value: Option<&str>, + ) -> CommonConfiguration { + let mut env_overrides = HashMap::new(); + let mut cli_overrides = BTreeMap::new(); + + if let Some(value) = override_value { + env_overrides.insert("PROPERTY".to_owned(), value.to_owned()); + cli_overrides.insert("--property".to_owned(), value.to_owned()); + } + + CommonConfiguration { + config, + config_overrides: TestConfigOverrides { + property: override_value.map(str::to_owned), + }, + env_overrides, + cli_overrides, + pod_overrides: PodTemplateSpec { + metadata: Some(ObjectMeta { + name: override_value.map(str::to_owned), + ..ObjectMeta::default() + }), + ..PodTemplateSpec::default() + }, + product_specific_common_config: ProductCommonConfig { + property: override_value.map(str::to_owned), + }, + } + } + + #[rstest] + #[case( + "role-group", + Some("role-group"), + Some("role-group"), + Some("role"), + Some("default") + )] + #[case( + "role-group", + Some("role-group"), + Some("role-group"), + Some("role"), + None + )] + #[case( + "role-group", + Some("role-group"), + Some("role-group"), + None, + Some("default") + )] + #[case("role-group", Some("role-group"), Some("role-group"), None, None)] + #[case("role", Some("role"), None, Some("role"), Some("default"))] + #[case("role", Some("role"), None, Some("role"), None)] + #[case("default", None, None, None, Some("default"))] + fn test_with_validated_config_and_result_ok( + #[case] expected_config_value: &str, + #[case] expected_override_value: Option<&str>, + #[case] role_group_value: Option<&str>, + #[case] role_value: Option<&str>, + #[case] default_value: Option<&str>, + ) { + let role_group = RoleGroup { + config: new_common_config(ConfigFragment::new(role_group_value), role_group_value), + replicas: Some(3), + }; + let role = Role::<_, _, GenericRoleConfig, _> { + config: new_common_config(ConfigFragment::new(role_value), role_value), + ..Role::default() + }; + let default_config = ConfigFragment::new(default_value); + + let result = with_validated_config(&role_group, &role, &default_config); + + assert_eq!( + Some(RoleGroup { + config: new_common_config( + Config::new(expected_config_value), + expected_override_value + ), + replicas: Some(3) + }), + result.ok() + ) + } + + #[test] + fn test_with_validated_config_and_result_err() { + let role_group = RoleGroup { + config: new_common_config(ConfigFragment::new(None), None), + replicas: None, + }; + let role = Role::<_, _, GenericRoleConfig, _> { + config: new_common_config(ConfigFragment::new(None), None), + ..Role::default() + }; + let default_config = ConfigFragment::new(None); + + let result: Result, _> = + with_validated_config(&role_group, &role, &default_config); + + assert!(result.is_err()) + } + + #[test] + fn test_resource_names() { + let resource_names = ResourceNames { + cluster_name: ClusterName::from_str_unsafe("my-cluster"), + product_name: ProductName::from_str_unsafe("my-product"), + }; + + assert_eq!( + ServiceAccountName::from_str_unsafe("my-cluster-serviceaccount"), + resource_names.service_account_name() + ); + assert_eq!( + RoleBindingName::from_str_unsafe("my-cluster-rolebinding"), + resource_names.role_binding_name() + ); + assert_eq!( + ClusterRoleName::from_str_unsafe("my-product-clusterrole"), + resource_names.cluster_role_name() + ); + } +} From d351a8a601c5670d19f484f5d32c3032b0ad3888 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:34:28 +0200 Subject: [PATCH 07/14] refactor: derive Ord/PartialOrd on AirflowRole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Required so AirflowRole can be used as a BTreeMap key in the new controller's validated cluster type. No behavioural change — the enum already derives Eq/Hash, and the variant ordering is stable. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/crd/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3e264113..2354b174 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -572,6 +572,8 @@ pub struct AirflowOpaConfig { pub cache: UserInformationCache, } +// REVIEW: Ord/PartialOrd added so AirflowRole can be used as a BTreeMap key +// in the new controller's ValidatedAirflowCluster (BTreeMap) #[derive( Clone, Debug, @@ -581,7 +583,9 @@ pub struct AirflowOpaConfig { Eq, Hash, JsonSchema, + Ord, PartialEq, + PartialOrd, Serialize, EnumString, )] From e231737f6271beb4672fc16fec93c13b35f4cfab Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:35:19 +0200 Subject: [PATCH 08/14] refactor: make stateful_set_service_name generic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function only calls rolegroup_ref.object_name() which is available on any RoleGroupRef. Making it generic allows the new controller to call it with RoleGroupRef. No behavioural change — existing callers continue to work. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/service.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index b9dbe325..67df3cec 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -4,6 +4,7 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, + kube::Resource, kvp::{Annotations, Labels, ObjectLabels}, role_utils::RoleGroupRef, }; @@ -106,8 +107,10 @@ pub fn build_rolegroup_metrics_service( }) } -pub fn stateful_set_service_name( - rolegroup_ref: &RoleGroupRef, +// REVIEW: made generic so the new controller can call this with +// RoleGroupRef instead of RoleGroupRef +pub fn stateful_set_service_name( + rolegroup_ref: &RoleGroupRef, ) -> Option { Some(rolegroup_headless_service_name( &rolegroup_ref.object_name(), From 56ebcd6b8f70cf6e48c50492a413386aba3641f2 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 10:36:32 +0200 Subject: [PATCH 09/14] refactor: simplify build_airflow_template_envs signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the &ExecutorConfig parameter with a plain bool (vector_agent_enabled) — the function only ever read config.logging.enable_vector_agent from it. This decouples the function from the ExecutorConfig type, allowing the new controller to call it with a pre-validated bool without needing to reconstruct an ExecutorConfig. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/airflow_controller.rs | 2 +- rust/operator-binary/src/env_vars.rs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 54671bb3..2c9f0c0d 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1340,7 +1340,7 @@ fn build_executor_template_config_map( .add_env_vars(build_airflow_template_envs( airflow, env_overrides, - merged_executor_config, + merged_executor_config.logging.enable_vector_agent, metadata_database_connection_details, git_sync_resources, resolved_product_image, diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index b4828dbe..115cb4d9 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -18,8 +18,8 @@ use stackable_operator::{ use crate::{ crd::{ - AirflowExecutor, AirflowRole, ExecutorConfig, LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, LOG_CONFIG_DIR, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, + TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -373,10 +373,11 @@ fn static_envs( /// Return environment variables to be applied to the configuration map used in conjunction with /// the `kubernetesExecutor` worker. +// REVIEW: simplified from &ExecutorConfig — only config.logging.enable_vector_agent was used pub fn build_airflow_template_envs( airflow: &v1alpha2::AirflowCluster, env_overrides: &HashMap, - config: &ExecutorConfig, + vector_agent_enabled: bool, metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, git_sync_resources: &git_sync::v1alpha2::GitSyncResources, resolved_product_image: &ResolvedProductImage, @@ -434,7 +435,7 @@ pub fn build_airflow_template_envs( // _STACKABLE_POST_HOOK will contain a command to create a shutdown hook that will be // evaluated in the wrapper for each stackable spark container: this is necessary for pods // that are created and then terminated (we do a similar thing for spark-k8s). - if config.logging.enable_vector_agent { + if vector_agent_enabled { env.insert( "_STACKABLE_POST_HOOK".into(), EnvVar { From 0a8c8b04aa7f0f4fcb8814b1480d35f4c0da7383 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 11:07:36 +0200 Subject: [PATCH 10/14] refactor: replace monolithic controller with pipeline architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the monolithic reconcile_airflow with a five-stage pipeline: dereference → validate → build → apply → update_status. The build stage is infallible — all validation and fallible operations (config generation, PodBuilder/ContainerBuilder usage, logging validation) happen in the validate stage. Services and PDBs are now built inline rather than delegating to service.rs and operations/. Also updates product_logging.rs and controller_commons.rs to accept pre-validated types (ValidatedContainerLogConfigChoice) instead of raw Logging/ContainerLogConfig, making those functions infallible. Co-Authored-By: Claude Opus 4.6 --- .../operator-binary/src/airflow_controller.rs | 3665 +++++++++++------ .../operator-binary/src/controller_commons.rs | 27 +- rust/operator-binary/src/main.rs | 3 +- rust/operator-binary/src/product_logging.rs | 69 +- 4 files changed, 2496 insertions(+), 1268 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 2c9f0c0d..aea53f8f 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,21 +1,23 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`] +//! +//! Pipeline architecture: dereference -> validate -> build -> apply -> update_status + +// --------------------------------------------------------------------------- +// Imports +// --------------------------------------------------------------------------- + use std::{ collections::{BTreeMap, BTreeSet, HashMap}, - io::Write, + marker::PhantomData, str::FromStr, sync::Arc, }; use const_format::concatcp; -use product_config::{ - ProductConfigManager, - flask_app_config_writer::{self, FlaskAppConfigWriterError}, - types::PropertyNameKind, -}; +use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ - self, configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ @@ -23,37 +25,37 @@ use stackable_operator::{ container::ContainerBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, - volume::{ - ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, - ListenerReference, VolumeBuilder, - }, + volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, }, }, cli::OperatorEnvironmentOptions, - cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, + cluster_resources::{ClusterResource, ClusterResourceApplyStrategy, ClusterResources}, commons::{ - product_image_selection::{self, ResolvedProductImage}, - random_secret_creation, + affinity::StackableAffinity, + product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, + resources::{NoRuntimeLimits, Resources}, }, - crd::{authentication::ldap, git_sync, listener}, - database_connections::{ - TemplatingMechanism, - drivers::{ - celery::CeleryDatabaseConnectionDetails, - sqlalchemy::SqlAlchemyDatabaseConnectionDetails, - }, + crd::{git_sync, listener}, + database_connections::drivers::{ + celery::CeleryDatabaseConnectionDetails, sqlalchemy::SqlAlchemyDatabaseConnectionDetails, }, k8s_openapi::{ - self, DeepMerge, + DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, PersistentVolumeClaim, PodTemplateSpec, Probe, ServiceAccount, - TCPSocketAction, + Affinity, ConfigMap, Container as K8sContainer, ContainerPort, EnvVar, + PersistentVolumeClaim, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, + ResourceRequirements, Service, ServiceAccount, ServicePort, ServiceSpec, + TCPSocketAction, Volume, VolumeMount, }, + policy::v1::PodDisruptionBudget, + rbac::v1::RoleBinding, + }, + apimachinery::pkg::{ + api::resource::Quantity, apis::meta::v1::LabelSelector, util::intstr::IntOrString, }, - apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kube::{ Resource, ResourceExt, @@ -61,19 +63,14 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, - kvp::{Annotation, Label, LabelError, Labels, ObjectLabels}, + kvp::{Annotation, Annotations, Label, Labels, ObjectLabels}, logging::controller::ReconcilerError, product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from, - env_vars_from_rolegroup_config, transform_all_roles_to_config, + env_vars_from, env_vars_from_rolegroup_config, transform_all_roles_to_config, validate_all_roles_and_groups_config, }, - product_logging::{ - self, - framework::LoggingError, - spec::{ContainerLogConfig, Logging}, - }, - role_utils::{GenericRoleConfig, RoleGroupRef}, + product_logging::{self, framework::LoggingError, spec::Logging}, + role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, @@ -81,41 +78,46 @@ use stackable_operator::{ }, utils::COMMON_BASH_TRAP_FUNCTIONS, }; +use stackable_operator::commons::random_secret_creation; +use stackable_operator::product_logging::spec::ContainerLogConfig; use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; use crate::{ - config::{self, PYTHON_IMPORTS}, - controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ - self, AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, - AirflowConfigOptions, AirflowExecutor, AirflowExecutorCommonConfiguration, AirflowRole, - CONFIG_PATH, Container, ExecutorConfig, HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, - LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, - STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowExecutor, + AirflowRole, AirflowStorageConfig, CONFIG_PATH, Container, ExecutorConfig, HTTP_PORT, + HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, + METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, + TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, authorization::AirflowAuthorizationResolved, - build_recommended_labels, - internal_secret::{ - FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY, - }, + internal_secret::{FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY}, v1alpha2, }, - env_vars::{self, build_airflow_template_envs}, - operations::{ - graceful_shutdown::{ - add_airflow_graceful_shutdown_config, add_executor_graceful_shutdown_config, + controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + env_vars, + framework::{ + self, HasName, HasUid, NameIsValidLabelValue, + builder::meta::ownerreference_from_resource, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, + }, + types::{ + kubernetes::{ConfigMapName, NamespaceName, Uid}, + operator::ClusterName, }, - pdb::add_pdbs, }, product_logging::extend_config_map_with_log_config, - service::{ - build_rolegroup_headless_service, build_rolegroup_metrics_service, - stateful_set_service_name, - }, + service::stateful_set_service_name, }; +// --------------------------------------------------------------------------- +// Constants and context +// --------------------------------------------------------------------------- + pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; pub const CONTAINER_IMAGE_BASE_NAME: &str = "airflow"; pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = @@ -127,256 +129,346 @@ pub struct Ctx { pub operator_environment: OperatorEnvironmentOptions, } -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -pub enum Error { - #[snafu(display("object defines no airflow config role"))] - NoAirflowRole, +// --------------------------------------------------------------------------- +// Validated types +// --------------------------------------------------------------------------- + +pub(crate) struct Prepared; +pub(crate) struct Applied; + +pub(crate) struct KubernetesResources { + pub stateful_sets: Vec, + pub config_maps: Vec, + pub services: Vec, + pub service_accounts: Vec, + pub role_bindings: Vec, + pub pod_disruption_budgets: Vec, + pub listeners: Vec, + pub _status: PhantomData, +} - #[snafu(display("failed to apply Service for {rolegroup}"))] - ApplyRoleGroupService { - source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, - }, +#[derive(Clone, Debug)] +pub struct ValidatedRoleConfig { + pub pdb_enabled: bool, + pub pdb_max_unavailable: Option, + pub listener_class: Option, + pub group_listener_name: Option, +} - #[snafu(display("failed to apply ConfigMap for {rolegroup}"))] - ApplyRoleGroupConfig { - source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, - }, +#[derive(Clone, Debug)] +pub struct ValidatedRoleGroupConfig { + pub resources: Resources, + pub logging: ValidatedLogging, + pub affinity: StackableAffinity, + pub graceful_shutdown_timeout: Duration, + pub config_file_content: String, +} - #[snafu(display("failed to apply StatefulSet for {rolegroup}"))] - ApplyRoleGroupStatefulSet { - source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, - }, +#[derive(Clone)] +pub struct PrecomputedPodData { + pub env_vars: Vec, + pub airflow_commands: Vec, + pub auth_volumes: Vec, + pub auth_volume_mounts: Vec, + pub extra_volumes: Vec, + pub extra_volume_mounts: Vec, + pub git_sync_containers: Vec, + pub git_sync_init_containers: Vec, + pub git_sync_volumes: Vec, + pub git_sync_volume_mounts: Vec, + pub vector_container: Option, + pub service_account_name: String, + pub replicas: Option, + pub pod_overrides: PodTemplateSpec, + pub executor: AirflowExecutor, + pub executor_template_configmap_name: Option, + pub listener_volume_claim_template: Option, +} - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, +#[derive(Clone, Debug)] +pub struct ValidatedLogging { + pub airflow_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, + pub git_sync_container_log_config: ContainerLogConfig, +} - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, +impl ValidatedLogging { + pub fn is_vector_agent_enabled(&self) -> bool { + self.vector_container.is_some() + } +} - #[snafu(display("Failed to transform configs"))] - ProductConfigTransform { - source: stackable_operator::product_config_utils::Error, - }, +// REVIEW: ValidatedAirflowCluster is the central validated type. It holds all data needed +// by the build stage so that build() can be infallible. All optional-after-merge fields +// are unwrapped during validation, and logging is pre-validated into ValidatedLogging. +#[derive(Clone)] +pub struct ValidatedAirflowCluster { + metadata: ObjectMeta, + pub image: ResolvedProductImage, + pub name: ClusterName, + pub namespace: NamespaceName, + pub uid: Uid, + pub role_groups: BTreeMap>, + pub precomputed_pod_data: BTreeMap>, + pub executor_template_config_maps: Vec, + pub role_configs: BTreeMap, + pub executor: AirflowExecutor, +} - #[snafu(display("failed to patch service account"))] - ApplyServiceAccount { - source: stackable_operator::cluster_resources::Error, - }, +impl ValidatedAirflowCluster { + #[allow(clippy::too_many_arguments)] + pub fn new( + image: ResolvedProductImage, + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + role_groups: BTreeMap>, + precomputed_pod_data: BTreeMap>, + executor_template_config_maps: Vec, + role_configs: BTreeMap, + executor: AirflowExecutor, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + image, + name, + namespace, + uid, + role_groups, + precomputed_pod_data, + executor_template_config_maps, + role_configs, + executor, + } + } - #[snafu(display("failed to patch role binding: {source}"))] - ApplyRoleBinding { - source: stackable_operator::cluster_resources::Error, - }, + pub fn rolegroup_ref(&self, role: &AirflowRole, role_group: &str) -> RoleGroupRef { + RoleGroupRef { + cluster: ObjectRef::from_obj(self), + role: role.to_string(), + role_group: role_group.to_string(), + } + } +} - #[snafu(display("failed to build RBAC objects"))] - BuildRBACObjects { - source: stackable_operator::commons::rbac::Error, - }, +impl HasName for ValidatedAirflowCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} - #[snafu(display("failed to build config file for {rolegroup}"))] - BuildRoleGroupConfigFile { - source: FlaskAppConfigWriterError, - rolegroup: RoleGroupRef, - }, +impl HasUid for ValidatedAirflowCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} - #[snafu(display("failed to build ConfigMap for {rolegroup}"))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, - }, +impl Resource for ValidatedAirflowCluster { + type DynamicType = + ::DynamicType; + type Scope = ::Scope; - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crd::Error }, + fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::kind(dt) + } - #[snafu(display("could not parse Airflow role [{role}]"))] - UnidentifiedAirflowRole { - source: strum::ParseError, - role: String, - }, + fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::group(dt) + } - #[snafu(display("invalid container name"))] - InvalidContainerName { - source: stackable_operator::builder::pod::container::Error, - }, + fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::version(dt) + } - #[snafu(display("invalid git-sync specification"))] - InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, + fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::plural(dt) + } - #[snafu(display("failed to create cluster resources"))] - CreateClusterResources { - source: stackable_operator::cluster_resources::Error, - }, + fn meta(&self) -> &ObjectMeta { + &self.metadata + } - #[snafu(display("failed to delete orphaned resources"))] - DeleteOrphanedResources { - source: stackable_operator::cluster_resources::Error, - }, + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, +impl NameIsValidLabelValue for ValidatedAirflowCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } +} - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, +// --------------------------------------------------------------------------- +// Error types and reconcile +// --------------------------------------------------------------------------- - #[snafu(display("failed to update status"))] - ApplyStatus { - source: stackable_operator::client::Error, +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("AirflowCluster object is invalid"))] + InvalidAirflowCluster { + source: error_boundary::InvalidObject, }, - #[snafu(display("failed to apply authentication configuration"))] - InvalidAuthenticationConfig { source: crd::authentication::Error }, - - #[snafu(display("pod template serialization"))] - PodTemplateSerde { source: serde_yaml::Error }, + #[snafu(display("failed to dereference resources"))] + Dereference { source: DereferenceError }, - #[snafu(display("failed to build the pod template config map"))] - PodTemplateConfigMap { - source: stackable_operator::builder::configmap::Error, - }, + #[snafu(display("failed to validate cluster"))] + Validate { source: ValidateError }, - #[snafu(display("failed to apply executor template ConfigMap"))] - ApplyExecutorTemplateConfig { + #[snafu(display("failed to create cluster resources"))] + CreateClusterResources { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to create PodDisruptionBudget"))] - FailedToCreatePdb { - source: crate::operations::pdb::Error, - }, + #[snafu(display("failed to apply resources"))] + Apply { source: ApplyError }, - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, - }, + #[snafu(display("failed to update status"))] + UpdateStatus { source: UpdateStatusError }, +} - #[snafu(display("failed to build label"))] - BuildLabel { source: LabelError }, +type Result = std::result::Result; - #[snafu(display("failed to build object meta data"))] - ObjectMeta { - source: stackable_operator::builder::meta::Error, - }, +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +// REVIEW: The reconcile pipeline is structured as five sequential stages: +// 1. dereference — async, fallible: resolve external references +// 2. validate — sync, fallible: validate and merge configs +// 3. build — sync, infallible: construct Kubernetes resources +// 4. apply — async, fallible: apply resources to the cluster +// 5. update_status — async, fallible: patch status on the CRD +pub async fn reconcile( + airflow: Arc>, + ctx: Arc, +) -> Result { + tracing::info!("Starting reconcile"); - #[snafu(display("failed to construct config"))] - ConstructConfig { source: config::Error }, + let airflow = airflow + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidAirflowClusterSnafu)?; - #[snafu(display( - "failed to write to String (Vec to be precise) containing Airflow config" - ))] - WriteToConfigFileString { source: std::io::Error }, + // --- dereference (async, fallible) --- + let dereferenced = dereference( + &ctx.client, + airflow, + CONTAINER_IMAGE_BASE_NAME, + &ctx.operator_environment.image_repository, + crate::built_info::PKG_VERSION, + ) + .await + .context(DereferenceSnafu)?; - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, + // --- validate (sync, fallible) --- + let validated = + validate_cluster(airflow, &dereferenced, &ctx.product_config).context(ValidateSnafu)?; - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, + // REVIEW: build() is infallible — all validation and fallible operations (config + // generation, PodBuilder/ContainerBuilder usage, logging validation) happen in the + // validate stage. The build stage purely assembles Kubernetes resource structs. + // --- build (sync, infallible) --- + let prepared = build(&validated); - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, - }, + // --- apply (async, fallible) --- + let cluster_resources = ClusterResources::new( + APP_NAME, + OPERATOR_NAME, + AIRFLOW_CONTROLLER_NAME, + &airflow.object_ref(&()), + ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), + &airflow.spec.object_overrides, + ) + .context(CreateClusterResourcesSnafu)?; - #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] - AddLdapVolumesAndVolumeMounts { source: ldap::v1alpha1::Error }, + let applied = Applier::new(&ctx.client, cluster_resources) + .apply(prepared) + .await + .context(ApplySnafu)?; - #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] - AddTlsVolumesAndVolumeMounts { - source: stackable_operator::commons::tls_verification::TlsClientDetailsError, - }, + // --- update status (async, fallible) --- + update_status(&ctx.client, airflow, applied) + .await + .context(UpdateStatusSnafu)?; - #[snafu(display("AirflowCluster object is invalid"))] - InvalidAirflowCluster { - source: error_boundary::InvalidObject, - }, + Ok(Action::await_change()) +} - #[snafu(display("failed to build Statefulset environmental variables"))] - BuildStatefulsetEnvVars { source: env_vars::Error }, +pub fn error_policy( + _obj: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidAirflowCluster { .. } => Action::await_change(), + _ => Action::requeue(*Duration::from_secs(10)), + } +} - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, +// --------------------------------------------------------------------------- +// Dereference +// --------------------------------------------------------------------------- - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum DereferenceError { + #[snafu(display("failed to resolve product image"))] + ResolveProductImage { + source: stackable_operator::commons::product_image_selection::Error, }, - #[snafu(display("failed to apply group listener"))] - ApplyGroupListener { - source: stackable_operator::cluster_resources::Error, + #[snafu(display("failed to apply authentication configuration"))] + InvalidAuthenticationConfig { + source: crate::crd::authentication::Error, }, - #[snafu(display("failed to configure service"))] - ServiceConfiguration { source: crate::service::Error }, - #[snafu(display("invalid authorization config"))] InvalidAuthorizationConfig { source: stackable_operator::commons::opa::Error, }, - #[snafu(display("failed to resolve product image"))] - ResolveProductImage { - source: product_image_selection::Error, - }, - #[snafu(display("failed to create internal secret"))] InvalidInternalSecret { source: random_secret_creation::Error, }, } -type Result = std::result::Result; - -impl ReconcilerError for Error { - fn category(&self) -> &'static str { - ErrorDiscriminants::from(self).into() - } +pub struct DereferencedObjects { + pub resolved_product_image: ResolvedProductImage, + pub authentication_config: AirflowClientAuthenticationDetailsResolved, + pub authorization_config: AirflowAuthorizationResolved, } -pub async fn reconcile_airflow( - airflow: Arc>, - ctx: Arc, -) -> Result { - tracing::info!("Starting reconcile"); - - let airflow = airflow - .0 - .as_ref() - .map_err(error_boundary::InvalidObject::clone) - .context(InvalidAirflowClusterSnafu)?; - - let client = &ctx.client; +pub async fn dereference( + client: &stackable_operator::client::Client, + airflow: &v1alpha2::AirflowCluster, + image_base_name: &str, + image_repository: &str, + pkg_version: &str, +) -> std::result::Result { let resolved_product_image = airflow .spec .image - .resolve( - CONTAINER_IMAGE_BASE_NAME, - &ctx.operator_environment.image_repository, - crate::built_info::PKG_VERSION, - ) - .context(ResolveProductImageSnafu)?; - - let cluster_operation_cond_builder = - ClusterOperationsConditionBuilder::new(&airflow.spec.cluster_operation); + .resolve(image_base_name, image_repository, pkg_version) + .context(deref_err::ResolveProductImageSnafu)?; let authentication_config = AirflowClientAuthenticationDetailsResolved::from( &airflow.spec.cluster_config.authentication, client, ) .await - .context(InvalidAuthenticationConfigSnafu)?; + .context(deref_err::InvalidAuthenticationConfigSnafu)?; let authorization_config = AirflowAuthorizationResolved::from_authorization_config( client, @@ -384,454 +476,315 @@ pub async fn reconcile_airflow( &airflow.spec.cluster_config.authorization, ) .await - .context(InvalidAuthorizationConfigSnafu)?; - // We don't have a config file, but do everything via env substitution - - let templating_mechanism = TemplatingMechanism::BashEnvSubstitution; - let metadata_database_connection_details = airflow - .spec - .cluster_config - .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + .context(deref_err::InvalidAuthorizationConfigSnafu)?; - let celery_database_connection_details = if let ( - Some(celery_results_backend), - Some(celery_broker), - ) = ( - &airflow.spec.cluster_config.celery_results_backend, - &airflow.spec.cluster_config.celery_broker, - ) { - // The celery results backend and celery broker only work with configured celeryExecutors. - // Emit a warning if celery executors were not configured properly. - if !matches!( - &airflow.spec.executor, - AirflowExecutor::CeleryExecutors { .. } - ) { - tracing::warn!( - "No `spec.celeryExecutors` configured, but `spec.clusterConfig.celeryResultsBackend` and `spec.clusterConfig.celeryBroker` are provided. This only works in combination with a celery executor!" - ) - } + random_secret_creation::create_random_secret_if_not_exists( + &airflow.shared_internal_secret_secret_name(), + INTERNAL_SECRET_SECRET_KEY, + 256, + airflow, + client, + ) + .await + .context(deref_err::InvalidInternalSecretSnafu)?; - let celery_results_backend = celery_results_backend - .celery_connection_details_with_templating( - "CELERY_RESULT_BACKEND", - &templating_mechanism, - ); - let celery_broker = celery_broker - .celery_connection_details_with_templating("CELERY_BROKER", &templating_mechanism); - Some((celery_results_backend, celery_broker)) - } else { - None - }; + random_secret_creation::create_random_secret_if_not_exists( + &airflow.shared_jwt_secret_secret_name(), + JWT_SECRET_SECRET_KEY, + 256, + airflow, + client, + ) + .await + .context(deref_err::InvalidInternalSecretSnafu)?; - let mut roles = HashMap::new(); + random_secret_creation::create_random_secret_if_not_exists( + &airflow.shared_fernet_key_secret_name(), + FERNET_KEY_SECRET_KEY, + 32, + airflow, + client, + ) + .await + .context(deref_err::InvalidInternalSecretSnafu)?; - // if the kubernetes executor is specified there will be no worker role as the pods - // are provisioned by airflow as defined by the task (default: one pod per task) - for role in AirflowRole::iter() { - if let Some(resolved_role) = airflow.get_role(&role) { - roles.insert( - role.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.into()), - ], - resolved_role.clone(), - ), - ); - } - } + Ok(DereferencedObjects { + resolved_product_image, + authentication_config, + authorization_config, + }) +} - let role_config = transform_all_roles_to_config(airflow, &roles); - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config.context(ProductConfigTransformSnafu)?, - &ctx.product_config, - false, - false, - ) - .context(InvalidProductConfigSnafu)?; +/// Module-like namespace for Snafu context selectors for DereferenceError, +/// avoiding name collisions with the top-level and validate error selectors. +mod deref_err { + pub(super) use super::{ + InvalidAuthenticationConfigSnafu, InvalidAuthorizationConfigSnafu, + InvalidInternalSecretSnafu, ResolveProductImageSnafu, + }; +} - let mut cluster_resources = ClusterResources::new( - APP_NAME, - OPERATOR_NAME, - AIRFLOW_CONTROLLER_NAME, - &airflow.object_ref(&()), - ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), - &airflow.spec.object_overrides, - ) - .context(CreateClusterResourcesSnafu)?; +// --------------------------------------------------------------------------- +// Validate +// --------------------------------------------------------------------------- - let required_labels = cluster_resources - .get_required_labels() - .context(BuildLabelSnafu)?; +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum ValidateError { + #[snafu(display("failed to validate cluster name"))] + InvalidClusterName { + source: crate::framework::macros::attributed_string_type::Error, + }, - let (rbac_sa, rbac_rolebinding) = - build_rbac_resources(airflow, APP_NAME, required_labels).context(BuildRBACObjectsSnafu)?; + #[snafu(display("object has no associated namespace"))] + ObjectHasNoNamespace, - let rbac_sa = cluster_resources - .add(client, rbac_sa.clone()) - .await - .context(ApplyServiceAccountSnafu)?; - cluster_resources - .add(client, rbac_rolebinding) - .await - .context(ApplyRoleBindingSnafu)?; + #[snafu(display("failed to validate cluster namespace"))] + InvalidClusterNamespace { + source: crate::framework::macros::attributed_string_type::Error, + }, - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + #[snafu(display("object has no UID"))] + ObjectHasNoUid, - let airflow_executor = &airflow.spec.executor; + #[snafu(display("failed to validate cluster UID"))] + InvalidClusterUid { + source: crate::framework::macros::attributed_string_type::Error, + }, - // if the kubernetes executor is specified, in place of a worker role that will be in the role - // collection there will be a pod template created to be used for pod provisioning - if let AirflowExecutor::KubernetesExecutors { - common_configuration, - } = &airflow_executor - { - build_executor_template( - airflow, - common_configuration, - &metadata_database_connection_details, - &resolved_product_image, - &authentication_config, - &authorization_config, - &mut cluster_resources, - client, - &rbac_sa, - ) - .await?; - } + #[snafu(display("failed to validate logging configuration"))] + ValidateLoggingConfig { + source: crate::framework::product_logging::framework::Error, + }, - random_secret_creation::create_random_secret_if_not_exists( - &airflow.shared_internal_secret_secret_name(), - INTERNAL_SECRET_SECRET_KEY, - 256, - airflow, - client, - ) - .await - .context(InvalidInternalSecretSnafu)?; + #[snafu(display("vectorAggregatorConfigMapName must be set when vector agent is enabled"))] + MissingVectorAggregatorConfigMapName, - random_secret_creation::create_random_secret_if_not_exists( - &airflow.shared_jwt_secret_secret_name(), - JWT_SECRET_SECRET_KEY, - 256, - airflow, - client, - ) - .await - .context(InvalidInternalSecretSnafu)?; + #[snafu(display("failed to parse vector aggregator ConfigMap name"))] + ParseVectorAggregatorConfigMapName { + source: crate::framework::macros::attributed_string_type::Error, + }, - random_secret_creation::create_random_secret_if_not_exists( - &airflow.shared_fernet_key_secret_name(), - FERNET_KEY_SECRET_KEY, - // https://airflow.apache.org/docs/apache-airflow/stable/security/secrets/fernet.html#security-fernet - // does not document how long the fernet key should be, but recommends using - // python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" - // which returns `jUm21LuA76YZmrIa9u4eXRg0h0P24MDC9IDOmDvJbfw=`, which has 44 characters, which makes 32 bytes. - 32, - airflow, - client, - ) - .await - .context(InvalidInternalSecretSnafu)?; + #[snafu(display("graceful shutdown timeout is not configured"))] + MissingGracefulShutdownTimeout, - for (role_name, role_config) in validated_role_config.iter() { - let airflow_role = - AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { - role: role_name.to_string(), - })?; + #[snafu(display("failed to resolve and merge config for role and role group"))] + FailedToResolveConfig { source: crate::crd::Error }, - if let Some(GenericRoleConfig { - pod_disruption_budget: pdb, - }) = airflow.role_config(&airflow_role) - { - add_pdbs(&pdb, airflow, &airflow_role, client, &mut cluster_resources) - .await - .context(FailedToCreatePdbSnafu)?; - } + #[snafu(display("failed to construct Airflow configuration"))] + ConstructConfig { source: crate::config::Error }, - if let Some(listener_class) = airflow_role.listener_class_name(airflow) { - if let Some(listener_group_name) = airflow.group_listener_name(&airflow_role) { - let rg_group_listener = build_group_listener( - airflow, - build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - role_name, - "none", - ), - listener_class.to_string(), - listener_group_name, - )?; - cluster_resources - .add(client, rg_group_listener) - .await - .context(ApplyGroupListenerSnafu)?; - } - } + #[snafu(display("failed to write config file"))] + BuildConfigFile { + source: product_config::flask_app_config_writer::FlaskAppConfigWriterError, + }, - for (rolegroup_name, rolegroup_config) in role_config.iter() { - let rolegroup = RoleGroupRef { - cluster: ObjectRef::from_obj(airflow), - role: role_name.into(), - role_group: rolegroup_name.into(), - }; + #[snafu(display("Failed to transform configs"))] + ProductConfigTransform { + source: stackable_operator::product_config_utils::Error, + }, - let merged_airflow_config = airflow - .merged_config(&airflow_role, &rolegroup) - .context(FailedToResolveConfigSnafu)?; - - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( - &airflow.spec.cluster_config.dags_git_sync, - &resolved_product_image, - &env_vars_from_rolegroup_config(rolegroup_config), - &airflow.volume_mounts(), - LOG_VOLUME_NAME, - &merged_airflow_config - .logging - .for_container(&Container::GitSync), - ) - .context(InvalidGitSyncSpecSnafu)?; + #[snafu(display("invalid product config"))] + InvalidProductConfig { + source: stackable_operator::product_config_utils::Error, + }, - let role_group_service_recommended_labels = build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - ); + #[snafu(display("could not parse Airflow role [{role}]"))] + UnidentifiedAirflowRole { + source: strum::ParseError, + role: String, + }, - let role_group_service_selector = Labels::role_group_selector( - airflow, - APP_NAME, - &rolegroup.role, - &rolegroup.role_group, - ) - .context(LabelBuildSnafu)?; + #[snafu(display("object defines no airflow config role"))] + NoAirflowRole, - let rg_headless_service = build_rolegroup_headless_service( - airflow, - &rolegroup, - role_group_service_recommended_labels.clone(), - role_group_service_selector.clone().into(), - ) - .context(ServiceConfigurationSnafu)?; + #[snafu(display("failed to build environment variables"))] + BuildEnvVars { source: crate::env_vars::Error }, - cluster_resources - .add(client, rg_headless_service) - .await - .context(ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + #[snafu(display("invalid git-sync specification"))] + InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, - let rg_metrics_service = build_rolegroup_metrics_service( - airflow, - &rolegroup, - role_group_service_recommended_labels, - role_group_service_selector.into(), - ) - .context(ServiceConfigurationSnafu)?; - cluster_resources - .add(client, rg_metrics_service) - .await - .context(ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + #[snafu(display("failed to configure logging"))] + ConfigureLogging { source: LoggingError }, - let rg_configmap = build_rolegroup_config_map( - airflow, - &resolved_product_image, - &rolegroup, - rolegroup_config, - &authentication_config, - &authorization_config, - &merged_airflow_config.logging, - &Container::Airflow, - )?; - cluster_resources - .add(client, rg_configmap) - .await - .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - })?; - - // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts - // to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - let rg_statefulset = build_server_rolegroup_statefulset( - airflow, - &resolved_product_image, - &airflow_role, - &rolegroup, - rolegroup_config, - &authentication_config, - &authorization_config, - &metadata_database_connection_details, - &celery_database_connection_details, - &rbac_sa, - &merged_airflow_config, - airflow_executor, - &git_sync_resources, - )?; + #[snafu(display("failed to add LDAP volumes and volume mounts"))] + AddLdapVolumesAndVolumeMounts { + source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, + }, - ss_cond_builder.add( - cluster_resources - .add(client, rg_statefulset) - .await - .context(ApplyRoleGroupStatefulSetSnafu { - rolegroup: rolegroup.clone(), - })?, - ); - } - } + #[snafu(display("failed to add TLS volumes and volume mounts"))] + AddTlsVolumesAndVolumeMounts { + source: stackable_operator::commons::tls_verification::TlsClientDetailsError, + }, - cluster_resources - .delete_orphaned_resources(client) - .await - .context(DeleteOrphanedResourcesSnafu)?; + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, + }, - let status = AirflowClusterStatus { - conditions: compute_conditions( - airflow, - &[&ss_cond_builder, &cluster_operation_cond_builder], - ), - }; + #[snafu(display("failed to build labels"))] + BuildLabels { + source: stackable_operator::kvp::LabelError, + }, - client - .apply_patch_status(OPERATOR_NAME, airflow, &status) - .await - .context(ApplyStatusSnafu)?; + #[snafu(display("invalid container name"))] + InvalidContainerName { + source: stackable_operator::builder::pod::container::Error, + }, - Ok(Action::await_change()) + #[snafu(display("failed to add volume mount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to add volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to serialize pod template"))] + PodTemplateSerde { source: serde_yaml::Error }, + + #[snafu(display("failed to build pod template ConfigMap"))] + PodTemplateConfigMap { + source: stackable_operator::builder::configmap::Error, + }, + + #[snafu(display("failed to build object metadata"))] + ObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build graceful shutdown config"))] + GracefulShutdown { + source: stackable_operator::builder::pod::Error, + }, } -#[allow(clippy::too_many_arguments)] -async fn build_executor_template( - airflow: &v1alpha2::AirflowCluster, - common_config: &AirflowExecutorCommonConfiguration, - metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - resolved_product_image: &ResolvedProductImage, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, - cluster_resources: &mut ClusterResources<'_>, - client: &stackable_operator::client::Client, - rbac_sa: &stackable_operator::k8s_openapi::api::core::v1::ServiceAccount, -) -> Result<(), Error> { - let merged_executor_config = airflow - .merged_executor_config(&common_config.config) - .context(FailedToResolveConfigSnafu)?; - let rolegroup = RoleGroupRef { - cluster: ObjectRef::from_obj(airflow), - role: "executor".into(), - role_group: "kubernetes".into(), +type ValidateResult = std::result::Result; + +fn validate_logging( + logging: &Logging, + main_container: Container, + vector_aggregator_config_map_name: Option<&str>, +) -> ValidateResult { + let airflow_container = validate_logging_configuration_for_container(logging, main_container) + .context(validate_err::ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let aggregator_name = vector_aggregator_config_map_name + .context(validate_err::MissingVectorAggregatorConfigMapNameSnafu)?; + ConfigMapName::from_str(aggregator_name) + .context(validate_err::ParseVectorAggregatorConfigMapNameSnafu)?; + let log_config = validate_logging_configuration_for_container(logging, Container::Vector) + .context(validate_err::ValidateLoggingConfigSnafu)?; + Some(VectorContainerLogConfig { log_config }) + } else { + None }; - let rg_configmap = build_rolegroup_config_map( - airflow, - resolved_product_image, - &rolegroup, - &HashMap::new(), - authentication_config, - authorization_config, - &merged_executor_config.logging, - &Container::Base, + let git_sync_container_log_config = logging.for_container(&Container::GitSync).into_owned(); + + Ok(ValidatedLogging { + airflow_container, + vector_container, + git_sync_container_log_config, + }) +} + +fn validate_airflow_config( + config: &AirflowConfig, + vector_aggregator_config_map_name: Option<&str>, + config_file_content: String, +) -> ValidateResult { + let logging = validate_logging( + &config.logging, + Container::Airflow, + vector_aggregator_config_map_name, )?; - cluster_resources - .add(client, rg_configmap) - .await - .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - })?; - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( - &airflow.spec.cluster_config.dags_git_sync, - resolved_product_image, - &env_vars_from(&common_config.env_overrides), - &airflow.volume_mounts(), - LOG_VOLUME_NAME, - &merged_executor_config - .logging - .for_container(&Container::GitSync), - ) - .context(InvalidGitSyncSpecSnafu)?; + let graceful_shutdown_timeout = config + .graceful_shutdown_timeout + .context(validate_err::MissingGracefulShutdownTimeoutSnafu)?; - let worker_pod_template_config_map = build_executor_template_config_map( - airflow, - resolved_product_image, - authentication_config, - metadata_database_connection_details, - &rbac_sa.name_unchecked(), - &merged_executor_config, - &common_config.env_overrides, - &common_config.pod_overrides, - &rolegroup, - &git_sync_resources, + Ok(ValidatedRoleGroupConfig { + resources: config.resources.clone(), + logging, + affinity: config.affinity.clone(), + graceful_shutdown_timeout, + config_file_content, + }) +} + +fn validate_executor_config( + config: &ExecutorConfig, + vector_aggregator_config_map_name: Option<&str>, + config_file_content: String, +) -> ValidateResult { + let logging = validate_logging( + &config.logging, + Container::Base, + vector_aggregator_config_map_name, )?; - cluster_resources - .add(client, worker_pod_template_config_map) - .await - .with_context(|_| ApplyExecutorTemplateConfigSnafu {})?; - Ok(()) + + let graceful_shutdown_timeout = config + .graceful_shutdown_timeout + .context(validate_err::MissingGracefulShutdownTimeoutSnafu)?; + + Ok(ValidatedRoleGroupConfig { + resources: config.resources.clone(), + logging, + affinity: config.affinity.clone(), + graceful_shutdown_timeout, + config_file_content, + }) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, +/// Generates the `webserver_config.py` content for a role group. +/// +/// This function is called during the validate stage so that the build stage can +/// construct ConfigMaps infallibly. +fn generate_config_file_content( authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, - logging: &Logging, - container: &Container, -) -> Result { - let mut config: BTreeMap = BTreeMap::new(); + product_version: &str, + rolegroup_config_overrides: &HashMap< + PropertyNameKind, + BTreeMap, + >, +) -> ValidateResult { + use std::io::Write; + + use product_config::flask_app_config_writer; + use stackable_operator::product_config_utils::{ + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, + }; + + use crate::{ + config::{self, PYTHON_IMPORTS}, + crd::{AIRFLOW_CONFIG_FILENAME, AirflowConfigOptions}, + }; - // this will call default values from AirflowClientAuthenticationDetails + let mut config = BTreeMap::new(); config::add_airflow_config( &mut config, authentication_config, authorization_config, - &resolved_product_image.product_version, + product_version, ) - .context(ConstructConfigSnafu)?; - - tracing::debug!( - "Default config for {}: {:?}", - rolegroup.object_name(), - config - ); + .context(validate_err::ConstructConfigSnafu)?; - let mut file_config = rolegroup_config + let mut file_overrides = rolegroup_config_overrides .get(&PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.to_string())) .cloned() .unwrap_or_default(); - tracing::debug!( - "Config overrides for {}: {:?}", - rolegroup.object_name(), - file_config - ); - - // now add any overrides, replacing any defaults - config.append(&mut file_config); - - tracing::debug!( - "Merged config for {}: {:?}", - rolegroup.object_name(), - config - ); + config.append(&mut file_overrides); let mut config_file = Vec::new(); - // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; + writeln!(config_file, "{}", header).expect("writing to Vec is infallible"); } let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); @@ -841,108 +794,1069 @@ fn build_rolegroup_config_map( config.iter(), PYTHON_IMPORTS, ) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; + .context(validate_err::BuildConfigFileSnafu)?; if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; + writeln!(config_file, "{}", footer).expect("writing to Vec is infallible"); } - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(rolegroup.object_name()) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(ObjectMetaSnafu)? - .build(), - ) - .add_data( - AIRFLOW_CONFIG_FILENAME, - String::from_utf8(config_file).unwrap(), - ); - - extend_config_map_with_log_config( - rolegroup, - logging, - container, - &Container::Vector, - &mut cm_builder, - resolved_product_image, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; + Ok(String::from_utf8(config_file).expect("flask_app_config_writer produces valid UTF-8")) +} - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) +/// Top-level validation: runs product_config, merges/validates per-rolegroup configs, +/// generates config file contents, and assembles a [`ValidatedAirflowCluster`]. +fn validate_cluster( + airflow: &v1alpha2::AirflowCluster, + dereferenced: &DereferencedObjects, + product_config_manager: &ProductConfigManager, +) -> ValidateResult { + let vector_aggregator_config_map_name = airflow + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref(); + + // --- product_config transform + validate --- + let mut roles = HashMap::new(); + for role in AirflowRole::iter() { + if let Some(resolved_role) = airflow.get_role(&role) { + roles.insert( + role.to_string(), + ( + vec![ + PropertyNameKind::Env, + PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.into()), + ], + resolved_role.clone(), + ), + ); + } + } + + let role_config = transform_all_roles_to_config(airflow, &roles); + let validated_role_config = validate_all_roles_and_groups_config( + &dereferenced.resolved_product_image.product_version, + &role_config.context(validate_err::ProductConfigTransformSnafu)?, + product_config_manager, + false, + false, + ) + .context(validate_err::InvalidProductConfigSnafu)?; + + // --- compute database connection details (infallible) --- + let templating_mechanism = + stackable_operator::database_connections::TemplatingMechanism::BashEnvSubstitution; + let metadata_database_connection_details = airflow + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let celery_database_connection_details = if let ( + Some(celery_results_backend), + Some(celery_broker), + ) = ( + &airflow.spec.cluster_config.celery_results_backend, + &airflow.spec.cluster_config.celery_broker, + ) { + // The celery results backend and celery broker only work with configured celeryExecutors. + // Emit a warning if celery executors were not configured properly. + if !matches!( + &airflow.spec.executor, + AirflowExecutor::CeleryExecutors { .. } + ) { + tracing::warn!( + "No `spec.celeryExecutors` configured, but `spec.clusterConfig.celeryResultsBackend` and `spec.clusterConfig.celeryBroker` are provided. This only works in combination with a celery executor!" + ) + } + let celery_results_backend = celery_results_backend + .celery_connection_details_with_templating( + "CELERY_RESULT_BACKEND", + &templating_mechanism, + ); + let celery_broker = celery_broker + .celery_connection_details_with_templating("CELERY_BROKER", &templating_mechanism); + Some((celery_results_backend, celery_broker)) + } else { + None + }; + + // --- compute auth volumes/mounts (fallible) --- + let (auth_volumes, auth_volume_mounts) = + compute_auth_volumes_and_mounts(&dereferenced.authentication_config)?; + + // --- service account name (matches build_rbac_resources output: "{cluster}-serviceaccount") --- + let service_account_name = format!("{}-serviceaccount", airflow.name_any()); + + // --- per-role/rolegroup validation --- + let mut validated_role_groups = BTreeMap::new(); + let mut all_precomputed_pod_data = BTreeMap::new(); + + for (role_name, role_config) in validated_role_config.iter() { + let airflow_role = + AirflowRole::from_str(role_name).context(validate_err::UnidentifiedAirflowRoleSnafu { + role: role_name.to_string(), + })?; + + let mut validated_groups = BTreeMap::new(); + let mut pod_data_groups = BTreeMap::new(); + + for (rolegroup_name, rolegroup_config) in role_config.iter() { + let rolegroup_ref = RoleGroupRef { + cluster: ObjectRef::from_obj(airflow), + role: role_name.into(), + role_group: rolegroup_name.into(), + }; + + let merged_airflow_config = airflow + .merged_config(&airflow_role, &rolegroup_ref) + .context(validate_err::FailedToResolveConfigSnafu)?; + + let config_file_content = generate_config_file_content( + &dereferenced.authentication_config, + &dereferenced.authorization_config, + &dereferenced.resolved_product_image.product_version, + rolegroup_config, + )?; + + let validated_config = validate_airflow_config( + &merged_airflow_config, + vector_aggregator_config_map_name, + config_file_content, + )?; + + let pod_data = compute_precomputed_pod_data( + airflow, + &airflow_role, + &rolegroup_ref, + rolegroup_config, + &dereferenced.resolved_product_image, + &dereferenced.authentication_config, + &dereferenced.authorization_config, + &metadata_database_connection_details, + &celery_database_connection_details, + &validated_config.logging, + &auth_volumes, + &auth_volume_mounts, + &service_account_name, + )?; + + validated_groups.insert(rolegroup_name.clone(), validated_config); + pod_data_groups.insert(rolegroup_name.clone(), pod_data); + } + + validated_role_groups.insert(airflow_role.clone(), validated_groups); + all_precomputed_pod_data.insert(airflow_role, pod_data_groups); + } + + // --- per-role config (PDB, listeners) --- + let mut validated_role_configs_map = BTreeMap::new(); + for role in AirflowRole::iter() { + if let Some(role_config) = airflow.role_config(&role) { + let pdb = &role_config.pod_disruption_budget; + let listener_class = role.listener_class_name(airflow); + let group_listener_name = airflow.group_listener_name(&role); + validated_role_configs_map.insert( + role, + ValidatedRoleConfig { + pdb_enabled: pdb.enabled, + pdb_max_unavailable: pdb.max_unavailable, + listener_class, + group_listener_name, + }, + ); + } + } + + // --- executor template config maps --- + let executor_template_config_maps = if let AirflowExecutor::KubernetesExecutors { + common_configuration, + } = &airflow.spec.executor + { + let merged_executor_config = airflow + .merged_executor_config(&common_configuration.config) + .context(validate_err::FailedToResolveConfigSnafu)?; + + let config_file_content = generate_config_file_content( + &dereferenced.authentication_config, + &dereferenced.authorization_config, + &dereferenced.resolved_product_image.product_version, + &HashMap::new(), + )?; + + let validated_config = validate_executor_config( + &merged_executor_config, + vector_aggregator_config_map_name, + config_file_content, + )?; + + build_executor_template_config_maps( + airflow, + &dereferenced.resolved_product_image, + &dereferenced.authentication_config, + &metadata_database_connection_details, + &service_account_name, + &validated_config, + common_configuration, + )? + } else { + Vec::new() + }; + + // --- assemble --- + validate_and_assemble( + airflow, + &dereferenced.resolved_product_image, + validated_role_groups, + all_precomputed_pod_data, + executor_template_config_maps, + validated_role_configs_map, + ) +} + +/// Validates the AirflowCluster and produces a [`ValidatedAirflowCluster`] containing +/// all role groups with their validated configs. +fn validate_and_assemble( + airflow: &v1alpha2::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + validated_role_configs: BTreeMap>, + precomputed_pod_data: BTreeMap>, + executor_template_config_maps: Vec, + role_configs: BTreeMap, +) -> ValidateResult { + let cluster_name = + ClusterName::from_str(&airflow.name_any()).context(validate_err::InvalidClusterNameSnafu)?; + let namespace = NamespaceName::from_str( + &airflow + .namespace() + .context(validate_err::ObjectHasNoNamespaceSnafu)?, + ) + .context(validate_err::InvalidClusterNamespaceSnafu)?; + let uid = Uid::from_str( + airflow + .meta() + .uid + .as_deref() + .context(validate_err::ObjectHasNoUidSnafu)?, + ) + .context(validate_err::InvalidClusterUidSnafu)?; + + Ok(ValidatedAirflowCluster::new( + resolved_product_image.clone(), + cluster_name, + namespace, + uid, + validated_role_configs, + precomputed_pod_data, + executor_template_config_maps, + role_configs, + airflow.spec.executor.clone(), + )) +} + +/// Extracts auth volumes and volume mounts using temporary builders. +/// +/// The upstream LDAP/TLS provider APIs require `PodBuilder`/`ContainerBuilder` references. +/// We create temporary builders, call the auth methods, then extract the raw volumes and mounts. +fn compute_auth_volumes_and_mounts( + authentication_config: &AirflowClientAuthenticationDetailsResolved, +) -> ValidateResult<(Vec, Vec)> { + let mut pb = PodBuilder::new(); + let mut cb = ContainerBuilder::new("dummy").expect("'dummy' is a valid container name"); + + let mut ldap_providers = BTreeSet::new(); + let mut tls_credentials = BTreeSet::new(); + + for auth_class in &authentication_config.authentication_classes_resolved { + match auth_class { + AirflowAuthenticationClassResolved::Ldap { provider } => { + ldap_providers.insert(provider); + } + AirflowAuthenticationClassResolved::Oidc { provider, .. } => { + tls_credentials.insert(&provider.tls); + } + } + } + + for provider in ldap_providers { + provider + .add_volumes_and_mounts(&mut pb, vec![&mut cb]) + .context(validate_err::AddLdapVolumesAndVolumeMountsSnafu)?; + } + for tls in tls_credentials { + tls.add_volumes_and_mounts(&mut pb, vec![&mut cb]) + .context(validate_err::AddTlsVolumesAndVolumeMountsSnafu)?; + } + + let container = cb.build(); + let pod_template = pb.build_template(); + + let volumes = pod_template + .spec + .and_then(|s| s.volumes) + .unwrap_or_default(); + let mounts = container.volume_mounts.unwrap_or_default(); + + Ok((volumes, mounts)) +} + +/// Builds the executor template ConfigMaps for KubernetesExecutor mode. +/// +/// Produces two ConfigMaps: +/// 1. A logging/config ConfigMap for the executor pods (equivalent to a rolegroup ConfigMap) +/// 2. A pod template ConfigMap containing a serialised PodTemplate that Airflow uses to +/// launch executor pods +/// +/// This is done in the validate stage because it uses PodBuilder/ContainerBuilder which +/// are fallible. The build stage then just passes these through to KubernetesResources. +#[allow(clippy::too_many_arguments)] +fn build_executor_template_config_maps( + airflow: &v1alpha2::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + authentication_config: &AirflowClientAuthenticationDetailsResolved, + metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, + service_account_name: &str, + validated_config: &ValidatedRoleGroupConfig, + common_configuration: &crate::crd::AirflowExecutorCommonConfiguration, +) -> ValidateResult> { + let executor_rolegroup_ref = RoleGroupRef { + cluster: ObjectRef::from_obj(airflow), + role: "executor".into(), + role_group: "kubernetes".into(), + }; + + // 1. Build the executor logging/config ConfigMap + let executor_config_cm = { + let metadata = ObjectMetaBuilder::new() + .name(executor_rolegroup_ref.object_name()) + .namespace_opt(airflow.namespace()) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(validate_err::ObjectMetaSnafu)? + .with_recommended_labels(&build_object_labels( + airflow, + resolved_product_image, + "executor", + "executor-template", + )) + .context(validate_err::ObjectMetaSnafu)? + .build(); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder.metadata(metadata); + cm_builder.add_data( + AIRFLOW_CONFIG_FILENAME, + validated_config.config_file_content.clone(), + ); + + extend_config_map_with_log_config( + &executor_rolegroup_ref, + &Container::Base, + &validated_config.logging.airflow_container, + validated_config.logging.vector_container.as_ref(), + &mut cm_builder, + resolved_product_image, + ); + + cm_builder + .build() + .context(validate_err::PodTemplateConfigMapSnafu)? + }; + + // 2. Build the executor pod template ConfigMap + let executor_template_cm = { + // git-sync resources for the executor template + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( + &airflow.spec.cluster_config.dags_git_sync, + resolved_product_image, + &env_vars_from(&common_configuration.env_overrides), + &airflow.volume_mounts(), + LOG_VOLUME_NAME, + &validated_config.logging.git_sync_container_log_config, + ) + .context(validate_err::InvalidGitSyncSpecSnafu)?; + + let mut pb = PodBuilder::new(); + let pb_metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&build_object_labels( + airflow, + resolved_product_image, + "executor", + "executor-template", + )) + .context(validate_err::ObjectMetaSnafu)? + .build(); + + pb.metadata(pb_metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .affinity(&validated_config.affinity) + .service_account_name(service_account_name) + .restart_policy("Never") + .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + + pb.termination_grace_period(&validated_config.graceful_shutdown_timeout) + .context(validate_err::GracefulShutdownSnafu)?; + + // Container name "base" is an Airflow requirement + let mut airflow_container = ContainerBuilder::new(&Container::Base.to_string()) + .context(validate_err::InvalidContainerNameSnafu)?; + + // Auth volumes and mounts + add_authentication_volumes_and_volume_mounts_to_builders( + authentication_config, + &mut airflow_container, + &mut pb, + )?; + + airflow_container + .image_from_product_image(resolved_product_image) + .resources(validated_config.resources.clone().into()) + .add_env_vars(env_vars::build_airflow_template_envs( + airflow, + &common_configuration.env_overrides, + validated_config.logging.is_vector_agent_enabled(), + metadata_database_connection_details, + &git_sync_resources, + resolved_product_image, + )) + .add_volume_mounts(airflow.volume_mounts()) + .context(validate_err::AddVolumeMountSnafu)? + .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) + .context(validate_err::AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR) + .context(validate_err::AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(validate_err::AddVolumeMountSnafu)?; + + // Git-sync resources (init containers only, no sidecars for executor template) + for container in git_sync_resources.git_sync_init_containers.iter().cloned() { + pb.add_init_container(container); + } + pb.add_volumes(git_sync_resources.git_content_volumes.clone()) + .context(validate_err::AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ssh_volumes.clone()) + .context(validate_err::AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ca_cert_volumes.clone()) + .context(validate_err::AddVolumeSnafu)?; + airflow_container + .add_volume_mounts(git_sync_resources.git_content_volume_mounts.clone()) + .context(validate_err::AddVolumeMountSnafu)?; + + // Database connection env vars + metadata_database_connection_details.add_to_container(&mut airflow_container); + + pb.add_container(airflow_container.build()); + pb.add_volumes(airflow.volumes().clone()) + .context(validate_err::AddVolumeSnafu)?; + // REVIEW: controller_commons::create_volumes now takes &ValidatedContainerLogConfigChoice + // instead of Option<&ContainerLogConfig>. The validated type ensures logging config + // has already been checked, so the build stage can use it directly. + pb.add_volumes(controller_commons::create_volumes( + &executor_rolegroup_ref.object_name(), + &validated_config.logging.airflow_container, + )) + .context(validate_err::AddVolumeSnafu)?; + + if let Some(vector_config) = &validated_config.logging.vector_container { + let vector_aggregator_config_map_name = airflow + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .context(validate_err::MissingVectorAggregatorConfigMapNameSnafu)?; + pb.add_container(build_logging_container( + resolved_product_image, + vector_config, + vector_aggregator_config_map_name, + )?); + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(common_configuration.pod_overrides.clone()); + + let restarter_label = Label::try_from(("restarter.stackable.tech/enabled", "true")) + .expect("static label is always valid"); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(airflow) + .name(airflow.executor_template_configmap_name()) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(validate_err::ObjectMetaSnafu)? + .with_recommended_labels(&build_object_labels( + airflow, + resolved_product_image, + "executor", + "executor-template", + )) + .context(validate_err::ObjectMetaSnafu)? + .with_label(restarter_label) + .build(), + ) + .add_data( + TEMPLATE_NAME, + serde_yaml::to_string(&pod_template) + .context(validate_err::PodTemplateSerdeSnafu)?, + ); + + cm_builder + .build() + .context(validate_err::PodTemplateConfigMapSnafu)? + }; + + Ok(vec![executor_config_cm, executor_template_cm]) +} + +/// Helper to add authentication volumes and volume mounts directly to builders. +/// Used by the executor template where we build a PodTemplate using PodBuilder/ContainerBuilder. +fn add_authentication_volumes_and_volume_mounts_to_builders( + authentication_config: &AirflowClientAuthenticationDetailsResolved, + cb: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> ValidateResult<()> { + let mut ldap_providers = BTreeSet::new(); + let mut tls_credentials = BTreeSet::new(); + + for auth_class in &authentication_config.authentication_classes_resolved { + match auth_class { + AirflowAuthenticationClassResolved::Ldap { provider } => { + ldap_providers.insert(provider); + } + AirflowAuthenticationClassResolved::Oidc { provider, .. } => { + tls_credentials.insert(&provider.tls); + } + } + } + + for provider in ldap_providers { + provider + .add_volumes_and_mounts(pb, vec![cb]) + .context(validate_err::AddLdapVolumesAndVolumeMountsSnafu)?; + } + for tls in tls_credentials { + tls.add_volumes_and_mounts(pb, vec![cb]) + .context(validate_err::AddTlsVolumesAndVolumeMountsSnafu)?; + } + Ok(()) +} + +fn build_object_labels<'a>( + airflow: &'a v1alpha2::AirflowCluster, + resolved_product_image: &'a ResolvedProductImage, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, v1alpha2::AirflowCluster> { + ObjectLabels { + owner: airflow, + app_name: APP_NAME, + app_version: &resolved_product_image.app_version_label_value, + operator_name: OPERATOR_NAME, + controller_name: AIRFLOW_CONTROLLER_NAME, + role, + role_group, + } } -fn build_rolegroup_metadata( +fn build_logging_container( + resolved_product_image: &ResolvedProductImage, + vector_config: &VectorContainerLogConfig, + vector_aggregator_config_map_name: &str, +) -> ValidateResult { + let raw_log_config = vector_config.log_config.to_raw_container_log_config(); + + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + Some(&raw_log_config), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(validate_err::ConfigureLoggingSnafu) +} + +/// Computes all pod-level data needed by the build stage to construct StatefulSets infallibly. +#[allow(clippy::too_many_arguments)] +fn compute_precomputed_pod_data( airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &&ResolvedProductImage, - rolegroup: &&RoleGroupRef, - prometheus_label: Label, - name: String, -) -> Result { + airflow_role: &AirflowRole, + rolegroup_ref: &RoleGroupRef, + rolegroup_config: &HashMap>, + resolved_product_image: &ResolvedProductImage, + authentication_config: &AirflowClientAuthenticationDetailsResolved, + authorization_config: &AirflowAuthorizationResolved, + metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, + celery_database_connection_details: &Option<( + CeleryDatabaseConnectionDetails, + CeleryDatabaseConnectionDetails, + )>, + validated_logging: &ValidatedLogging, + auth_volumes: &[Volume], + auth_volume_mounts: &[VolumeMount], + service_account_name: &str, +) -> ValidateResult { + let executor = &airflow.spec.executor; + + // --- git-sync resources --- + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( + &airflow.spec.cluster_config.dags_git_sync, + resolved_product_image, + &env_vars_from_rolegroup_config(rolegroup_config), + &airflow.volume_mounts(), + LOG_VOLUME_NAME, + &validated_logging.git_sync_container_log_config, + ) + .context(validate_err::InvalidGitSyncSpecSnafu)?; + + // --- env vars --- + let mut env_vars = env_vars::build_airflow_statefulset_envs( + airflow, + airflow_role, + rolegroup_config, + executor, + authentication_config, + authorization_config, + metadata_database_connection_details, + celery_database_connection_details, + &git_sync_resources, + resolved_product_image, + ) + .context(validate_err::BuildEnvVarsSnafu)?; + + // Database connection details add secret-referenced env vars via ContainerBuilder. + // Extract them using a temp builder. + let db_env_vars = { + let mut cb = ContainerBuilder::new("dummy").expect("'dummy' is a valid container name"); + metadata_database_connection_details.add_to_container(&mut cb); + if let Some((celery_result_backend, celery_broker)) = celery_database_connection_details { + celery_result_backend.add_to_container(&mut cb); + celery_broker.add_to_container(&mut cb); + } + cb.build().env.unwrap_or_default() + }; + env_vars.extend(db_env_vars); + + // --- commands --- + let airflow_commands = + airflow_role.get_commands(airflow, authentication_config, resolved_product_image); + + // --- git-sync containers/volumes --- + let use_git_sync_init_containers = matches!(executor, AirflowExecutor::CeleryExecutors { .. }); + let git_sync_containers = git_sync_resources.git_sync_containers.clone(); + let git_sync_init_containers = if use_git_sync_init_containers { + git_sync_resources.git_sync_init_containers.clone() + } else { + Vec::new() + }; + let mut git_sync_volumes = git_sync_resources.git_content_volumes.clone(); + git_sync_volumes.extend(git_sync_resources.git_ssh_volumes.clone()); + git_sync_volumes.extend(git_sync_resources.git_ca_cert_volumes.clone()); + let git_sync_volume_mounts = git_sync_resources.git_content_volume_mounts.clone(); + + // --- vector container --- + let vector_container = if let Some(vector_config) = &validated_logging.vector_container { + let vector_aggregator_config_map_name = airflow + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .context(validate_err::MissingVectorAggregatorConfigMapNameSnafu)?; + Some(build_logging_container( + resolved_product_image, + vector_config, + vector_aggregator_config_map_name, + )?) + } else { + None + }; + + // --- replicas --- + let binding = airflow.get_role(airflow_role); + let role = binding.as_ref().context(validate_err::NoAirflowRoleSnafu)?; + let rolegroup = role.role_groups.get(&rolegroup_ref.role_group); + let replicas = rolegroup.and_then(|rg| rg.replicas); + + // --- pod overrides --- + let mut pod_overrides = PodTemplateSpec::default(); + pod_overrides.merge_from(role.config.pod_overrides.clone()); + if let Some(rg) = rolegroup { + pod_overrides.merge_from(rg.config.pod_overrides.clone()); + } + + // --- executor template configmap name --- + let executor_template_configmap_name = + if matches!(executor, AirflowExecutor::KubernetesExecutors { .. }) { + Some(airflow.executor_template_configmap_name()) + } else { + None + }; + + // --- listener PVC --- + let listener_volume_claim_template = if airflow_role.get_http_port().is_some() { + if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) { + let unversioned_labels = Labels::recommended(&ObjectLabels { + owner: airflow, + app_name: APP_NAME, + app_version: "none", + operator_name: OPERATOR_NAME, + controller_name: AIRFLOW_CONTROLLER_NAME, + role: &rolegroup_ref.role, + role_group: &rolegroup_ref.role_group, + }) + .context(validate_err::BuildLabelsSnafu)?; + + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(listener_group_name), + &unversioned_labels, + ) + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(validate_err::BuildListenerVolumeSnafu)?; + Some(pvc) + } else { + None + } + } else { + None + }; + + // --- user-defined extra volumes/mounts from CRD --- + let extra_volumes = airflow.volumes().clone(); + let extra_volume_mounts = airflow.volume_mounts(); + + Ok(PrecomputedPodData { + env_vars, + airflow_commands, + auth_volumes: auth_volumes.to_vec(), + auth_volume_mounts: auth_volume_mounts.to_vec(), + extra_volumes, + extra_volume_mounts, + git_sync_containers, + git_sync_init_containers, + git_sync_volumes, + git_sync_volume_mounts, + vector_container, + service_account_name: service_account_name.to_string(), + replicas, + pod_overrides, + executor: executor.clone(), + executor_template_configmap_name, + listener_volume_claim_template, + }) +} + +/// Module-like namespace for Snafu context selectors for ValidateError, +/// avoiding name collisions with the top-level error selectors. +mod validate_err { + pub(super) use super::{ + AddLdapVolumesAndVolumeMountsSnafu, AddTlsVolumesAndVolumeMountsSnafu, AddVolumeMountSnafu, + AddVolumeSnafu, BuildConfigFileSnafu, BuildEnvVarsSnafu, BuildLabelsSnafu, + BuildListenerVolumeSnafu, ConfigureLoggingSnafu, ConstructConfigSnafu, + FailedToResolveConfigSnafu, GracefulShutdownSnafu, InvalidClusterNameSnafu, + InvalidClusterNamespaceSnafu, InvalidClusterUidSnafu, InvalidContainerNameSnafu, + InvalidGitSyncSpecSnafu, InvalidProductConfigSnafu, + MissingGracefulShutdownTimeoutSnafu, MissingVectorAggregatorConfigMapNameSnafu, + NoAirflowRoleSnafu, ObjectHasNoNamespaceSnafu, ObjectHasNoUidSnafu, ObjectMetaSnafu, + ParseVectorAggregatorConfigMapNameSnafu, PodTemplateConfigMapSnafu, + PodTemplateSerdeSnafu, ProductConfigTransformSnafu, UnidentifiedAirflowRoleSnafu, + ValidateLoggingConfigSnafu, + }; +} + +// --------------------------------------------------------------------------- +// Build (including RoleGroupBuilder) +// --------------------------------------------------------------------------- + +fn main_container_for_role(_role: &AirflowRole) -> Container { + Container::Airflow +} + +// REVIEW: build() is infallible. All validation and fallible operations (config generation, +// PodBuilder/ContainerBuilder usage, logging validation) are performed in the validate +// stage. The build stage purely assembles Kubernetes resource structs from validated data. +fn build(validated: &ValidatedAirflowCluster) -> KubernetesResources { + let mut stateful_sets = Vec::new(); + let mut config_maps = Vec::new(); + let mut services = Vec::new(); + let mut pod_disruption_budgets = Vec::new(); + let mut listeners = Vec::new(); + + // --- RBAC --- + let rbac_labels = build_recommended_labels(validated, "rbac", "rbac"); + + let (rbac_sa, rbac_rolebinding) = build_rbac_resources(validated, APP_NAME, rbac_labels) + .expect( + "RBAC resources should be created because the validated cluster has valid metadata", + ); + + // --- Executor template ConfigMaps (pre-built in validate stage) --- + config_maps.extend(validated.executor_template_config_maps.clone()); + + // --- Per-role/rolegroup resources --- + for (airflow_role, role_groups) in &validated.role_groups { + // PDBs + if let Some(role_config) = validated.role_configs.get(airflow_role) { + if let Some(pdb) = build_pdb(validated, airflow_role, role_config) { + pod_disruption_budgets.push(pdb); + } + } + + // Group listeners (only Webserver) + if let Some(role_config) = validated.role_configs.get(airflow_role) { + if let (Some(listener_class), Some(listener_name)) = ( + &role_config.listener_class, + &role_config.group_listener_name, + ) { + listeners.push(build_group_listener( + validated, + airflow_role, + listener_class.clone(), + listener_name.clone(), + )); + } + } + + for (rolegroup_name, role_group_config) in role_groups { + let rolegroup_ref = validated.rolegroup_ref(airflow_role, rolegroup_name); + + let main_container = main_container_for_role(airflow_role); + + // Services + services.push(build_headless_service(validated, &rolegroup_ref)); + services.push(build_metrics_service(validated, &rolegroup_ref)); + + // ConfigMap + StatefulSet via RoleGroupBuilder + let pod_data = validated + .precomputed_pod_data + .get(airflow_role) + .and_then(|groups| groups.get(rolegroup_name)) + .expect( + "PrecomputedPodData should exist for every role group \ + because validate_cluster computes it for each one", + ); + + let builder = RoleGroupBuilder::new( + validated, + role_group_config, + rolegroup_ref, + airflow_role.clone(), + main_container, + pod_data, + ); + + config_maps.push(builder.build_config_map()); + stateful_sets.push(builder.build_stateful_set()); + } + } + + KubernetesResources { + stateful_sets, + config_maps, + services, + service_accounts: vec![rbac_sa], + role_bindings: vec![rbac_rolebinding], + pod_disruption_budgets, + listeners, + _status: PhantomData, + } +} + +fn build_pdb( + cluster: &ValidatedAirflowCluster, + role: &AirflowRole, + role_config: &ValidatedRoleConfig, +) -> Option { + if !role_config.pdb_enabled { + return None; + } + + let max_unavailable = role_config.pdb_max_unavailable.unwrap_or(match role { + AirflowRole::Worker => match &cluster.executor { + AirflowExecutor::KubernetesExecutors { .. } => return None, + _ => 1, + }, + _ => 1, + }); + + // REVIEW: from_str_unsafe is used here because the values come from constants (APP_NAME, + // OPERATOR_NAME, AIRFLOW_CONTROLLER_NAME) or validated role names — they are known to be + // valid at compile time or have been validated during the validate stage. + Some({ + use crate::framework::types::operator::*; + framework::builder::pdb::pod_disruption_budget_builder_with_role( + cluster, + &ProductName::from_str_unsafe(APP_NAME), + &RoleName::from_str_unsafe(&role.to_string()), + &OperatorName::from_str_unsafe(OPERATOR_NAME), + &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), + ) + .with_max_unavailable(max_unavailable) + .build() + }) +} + +fn build_headless_service( + cluster: &ValidatedAirflowCluster, + rolegroup_ref: &RoleGroupRef, +) -> Service { let metadata = ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(name) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, + .name(format!("{}-headless", rolegroup_ref.object_name())) + .namespace(&cluster.namespace) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(build_recommended_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, )) - .context(ObjectMetaSnafu)? - .with_label(prometheus_label) .build(); - Ok(metadata) + + Service { + metadata, + spec: Some(ServiceSpec { + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(vec![ServicePort { + name: Some(HTTP_PORT_NAME.to_string()), + port: HTTP_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }]), + selector: Some( + build_role_group_selector_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } } -pub fn build_group_listener( - airflow: &v1alpha2::AirflowCluster, - object_labels: ObjectLabels, +fn build_metrics_service( + cluster: &ValidatedAirflowCluster, + rolegroup_ref: &RoleGroupRef, +) -> Service { + let metadata = ObjectMetaBuilder::new() + .name(format!("{}-metrics", rolegroup_ref.object_name())) + .namespace(&cluster.namespace) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(build_recommended_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .with_labels(prometheus_labels()) + .with_annotations(prometheus_annotations()) + .build(); + + Service { + metadata, + spec: Some(ServiceSpec { + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }]), + selector: Some( + build_role_group_selector_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +// REVIEW: from_str_unsafe is used for label construction throughout these helpers because +// the inputs are either compile-time constants (APP_NAME, OPERATOR_NAME, etc.) or values +// that have already been validated during the validate stage (role names, role group names, +// product versions). Using from_str_unsafe avoids redundant re-validation in the infallible +// build stage. +fn build_recommended_labels( + cluster: &ValidatedAirflowCluster, + role: &str, + role_group: &str, +) -> Labels { + use crate::framework::types::operator::*; + framework::kvp::label::recommended_labels( + cluster, + &ProductName::from_str_unsafe(APP_NAME), + &ProductVersion::from_str_unsafe(&cluster.image.app_version_label_value.to_string()), + &OperatorName::from_str_unsafe(OPERATOR_NAME), + &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), + &RoleName::from_str_unsafe(role), + &RoleGroupName::from_str_unsafe(role_group), + ) +} + +fn build_role_group_selector_labels( + cluster: &ValidatedAirflowCluster, + role: &str, + role_group: &str, +) -> Labels { + use crate::framework::types::operator::*; + framework::kvp::label::role_group_selector( + cluster, + &ProductName::from_str_unsafe(APP_NAME), + &RoleName::from_str_unsafe(role), + &RoleGroupName::from_str_unsafe(role_group), + ) +} + +fn prometheus_labels() -> Labels { + Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") +} + +fn prometheus_annotations() -> Annotations { + Annotations::try_from([ + ("prometheus.io/path".to_owned(), "/metrics".to_owned()), + ("prometheus.io/port".to_owned(), METRICS_PORT.to_string()), + ("prometheus.io/scheme".to_owned(), "http".to_owned()), + ("prometheus.io/scrape".to_owned(), "true".to_owned()), + ]) + .expect("should be valid annotations") +} + +fn build_group_listener( + cluster: &ValidatedAirflowCluster, + role: &AirflowRole, listener_class: String, listener_group_name: String, -) -> Result { - Ok(listener::v1alpha1::Listener { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(listener_group_name) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(ObjectMetaSnafu)? - .build(), +) -> listener::v1alpha1::Listener { + let metadata = ObjectMetaBuilder::new() + .name(&listener_group_name) + .namespace(&cluster.namespace) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(build_recommended_labels(cluster, &role.to_string(), "none")) + .build(); + + listener::v1alpha1::Listener { + metadata, spec: listener::v1alpha1::ListenerSpec { class_name: Some(listener_class), ports: Some(listener_ports()), ..listener::v1alpha1::ListenerSpec::default() }, status: None, - }) + } } -/// We only use the http port here and intentionally omit -/// the metrics one. fn listener_ports() -> Vec { vec![listener::v1alpha1::ListenerPort { name: HTTP_PORT_NAME.to_string(), @@ -951,561 +1865,906 @@ fn listener_ports() -> Vec { }] } -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -#[allow(clippy::too_many_arguments)] -fn build_server_rolegroup_statefulset( - airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - airflow_role: &AirflowRole, - rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, - metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - celery_database_connection_details: &Option<( - CeleryDatabaseConnectionDetails, - CeleryDatabaseConnectionDetails, - )>, - service_account: &ServiceAccount, - merged_airflow_config: &AirflowConfig, - executor: &AirflowExecutor, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, -) -> Result { - let binding = airflow.get_role(airflow_role); - let role = binding.as_ref().context(NoAirflowRoleSnafu)?; +// --------------------------------------------------------------------------- +// RoleGroupBuilder +// --------------------------------------------------------------------------- + +struct RoleGroupBuilder<'a> { + cluster: &'a ValidatedAirflowCluster, + role_group_config: &'a ValidatedRoleGroupConfig, + rolegroup_ref: RoleGroupRef, + airflow_role: AirflowRole, + main_container: Container, + pod_data: &'a PrecomputedPodData, +} + +impl<'a> RoleGroupBuilder<'a> { + fn new( + cluster: &'a ValidatedAirflowCluster, + role_group_config: &'a ValidatedRoleGroupConfig, + rolegroup_ref: RoleGroupRef, + airflow_role: AirflowRole, + main_container: Container, + pod_data: &'a PrecomputedPodData, + ) -> Self { + Self { + cluster, + role_group_config, + rolegroup_ref, + airflow_role, + main_container, + pod_data, + } + } + + fn build_config_map(&self) -> ConfigMap { + let metadata = self + .common_metadata(self.rolegroup_ref.object_name()) + .build(); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder.metadata(metadata); + + cm_builder.add_data( + AIRFLOW_CONFIG_FILENAME, + self.role_group_config.config_file_content.clone(), + ); + + extend_config_map_with_log_config( + &self.rolegroup_ref, + &self.main_container, + &self.role_group_config.logging.airflow_container, + self.role_group_config.logging.vector_container.as_ref(), + &mut cm_builder, + &self.cluster.image, + ); + + cm_builder + .build() + .expect("ConfigMap should build because metadata is set") + } + + fn build_stateful_set(&self) -> StatefulSet { + let restarter_label = Label::try_from(("restarter.stackable.tech/enabled", "true")) + .expect("static label is always valid"); + + let metadata = self + .common_metadata(self.rolegroup_ref.object_name()) + .with_label(restarter_label) + .build(); + + let template = self.build_pod_template(); + + let match_labels = { + use crate::framework::types::operator::*; + framework::kvp::label::role_group_selector( + self.cluster, + &ProductName::from_str_unsafe(APP_NAME), + &RoleName::from_str_unsafe(&self.rolegroup_ref.role), + &RoleGroupName::from_str_unsafe(&self.rolegroup_ref.role_group), + ) + }; + + let pod_management_policy = match self.airflow_role { + AirflowRole::Scheduler => "OrderedReady", + AirflowRole::Webserver + | AirflowRole::Worker + | AirflowRole::DagProcessor + | AirflowRole::Triggerer => "Parallel", + } + .to_string(); + + let spec = StatefulSetSpec { + pod_management_policy: Some(pod_management_policy), + replicas: self.pod_data.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some(match_labels.into()), + ..LabelSelector::default() + }, + service_name: stateful_set_service_name(&self.rolegroup_ref), + template, + volume_claim_templates: self + .pod_data + .listener_volume_claim_template + .clone() + .map(|pvc| vec![pvc]), + ..StatefulSetSpec::default() + }; + + StatefulSet { + metadata, + spec: Some(spec), + status: None, + } + } + + fn build_pod_template(&self) -> PodTemplateSpec { + let pod_metadata = ObjectMetaBuilder::new() + .with_labels(self.recommended_labels()) + .with_annotation( + Annotation::try_from(( + "kubectl.kubernetes.io/default-container", + format!("{}", self.main_container), + )) + .expect("static annotation is always valid"), + ) + .build(); + + let airflow_container = self.build_airflow_container(); + let metrics_container = self.build_metrics_container(); + + let mut containers = vec![airflow_container, metrics_container]; + containers.extend(self.pod_data.git_sync_containers.clone()); + if let Some(vector_container) = &self.pod_data.vector_container { + containers.push(vector_container.clone()); + } + + let init_containers = if self.pod_data.git_sync_init_containers.is_empty() { + None + } else { + Some(self.pod_data.git_sync_init_containers.clone()) + }; + + let volumes = self.build_volumes(); + + let termination_grace_period_seconds = self + .role_group_config + .graceful_shutdown_timeout + .as_secs() + .try_into() + .ok(); + + let mut pod_template = PodTemplateSpec { + metadata: Some(pod_metadata), + spec: Some(PodSpec { + affinity: { + let a = &self.role_group_config.affinity; + if a.pod_affinity.is_some() + || a.pod_anti_affinity.is_some() + || a.node_affinity.is_some() + { + Some(Affinity { + pod_affinity: a.pod_affinity.clone(), + pod_anti_affinity: a.pod_anti_affinity.clone(), + node_affinity: a.node_affinity.clone(), + }) + } else { + None + } + }, + containers, + init_containers, + service_account_name: Some(self.pod_data.service_account_name.clone()), + termination_grace_period_seconds, + security_context: Some(PodSecurityContext { + fs_group: Some(1000), + ..PodSecurityContext::default() + }), + image_pull_secrets: self.cluster.image.pull_secrets.clone(), + volumes: if volumes.is_empty() { + None + } else { + Some(volumes) + }, + ..PodSpec::default() + }), + }; + + pod_template.merge_from(self.pod_data.pod_overrides.clone()); + pod_template + } - let rolegroup = role.role_groups.get(&rolegroup_ref.role_group); + fn build_airflow_container(&self) -> K8sContainer { + let mut volume_mounts = vec![ + VolumeMount { + name: CONFIG_VOLUME_NAME.to_string(), + mount_path: CONFIG_PATH.to_string(), + ..VolumeMount::default() + }, + VolumeMount { + name: LOG_CONFIG_VOLUME_NAME.to_string(), + mount_path: LOG_CONFIG_DIR.to_string(), + ..VolumeMount::default() + }, + VolumeMount { + name: LOG_VOLUME_NAME.to_string(), + mount_path: STACKABLE_LOG_DIR.to_string(), + ..VolumeMount::default() + }, + ]; - let mut pb = PodBuilder::new(); - let recommended_object_labels = build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(LabelBuildSnafu)?; - - let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&recommended_object_labels) - .context(ObjectMetaSnafu)? - .with_annotation( - Annotation::try_from(( - "kubectl.kubernetes.io/default-container", - format!("{}", Container::Airflow), - )) - .expect("static annotation is always valid"), - ) - .build(); + volume_mounts.extend(self.pod_data.extra_volume_mounts.clone()); + volume_mounts.extend(self.pod_data.auth_volume_mounts.clone()); + volume_mounts.extend(self.pod_data.git_sync_volume_mounts.clone()); - pb.metadata(pb_metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .affinity(&merged_airflow_config.affinity) - .service_account_name(service_account.name_any()) - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + if matches!( + self.pod_data.executor, + AirflowExecutor::KubernetesExecutors { .. } + ) { + volume_mounts.push(VolumeMount { + name: TEMPLATE_VOLUME_NAME.to_string(), + mount_path: TEMPLATE_LOCATION.to_string(), + ..VolumeMount::default() + }); + } - let mut airflow_container = ContainerBuilder::new(&Container::Airflow.to_string()) - .context(InvalidContainerNameSnafu)?; + if self.airflow_role.get_http_port().is_some() + && self.pod_data.listener_volume_claim_template.is_some() + { + volume_mounts.push(VolumeMount { + name: LISTENER_VOLUME_NAME.to_string(), + mount_path: LISTENER_VOLUME_DIR.to_string(), + ..VolumeMount::default() + }); + } - add_authentication_volumes_and_volume_mounts( - authentication_config, - &mut airflow_container, - &mut pb, - )?; + let mut ports = Vec::new(); + if let Some(http_port) = self.airflow_role.get_http_port() { + ports.push(ContainerPort { + name: Some(HTTP_PORT_NAME.to_string()), + container_port: http_port.into(), + ..ContainerPort::default() + }); + } - add_airflow_graceful_shutdown_config(merged_airflow_config, &mut pb) - .context(GracefulShutdownSnafu)?; + let (readiness_probe, liveness_probe) = + if let Some(http_port) = self.airflow_role.get_http_port() { + let probe = Probe { + tcp_socket: Some(TCPSocketAction { + port: IntOrString::Int(http_port.into()), + ..TCPSocketAction::default() + }), + initial_delay_seconds: Some(60), + period_seconds: Some(10), + failure_threshold: Some(6), + ..Probe::default() + }; + (Some(probe.clone()), Some(probe)) + } else { + (None, None) + }; - let mut airflow_container_args = Vec::new(); - airflow_container_args.extend(airflow_role.get_commands( - airflow, - authentication_config, - resolved_product_image, - )); - - airflow_container - .image_from_product_image(resolved_product_image) - .resources(merged_airflow_config.resources.clone().into()) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![airflow_container_args.join("\n")]); - - airflow_container.add_env_vars( - env_vars::build_airflow_statefulset_envs( - airflow, - airflow_role, - rolegroup_config, - executor, - authentication_config, - authorization_config, - metadata_database_connection_details, - celery_database_connection_details, - git_sync_resources, - resolved_product_image, - ) - .context(BuildStatefulsetEnvVarsSnafu)?, - ); - - let volume_mounts = airflow.volume_mounts(); - airflow_container - .add_volume_mounts(volume_mounts) - .context(AddVolumeMountSnafu)?; - airflow_container - .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) - .context(AddVolumeMountSnafu)?; - airflow_container - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR) - .context(AddVolumeMountSnafu)?; - airflow_container - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)?; - - if let AirflowExecutor::KubernetesExecutors { .. } = executor { - airflow_container - .add_volume_mount(TEMPLATE_VOLUME_NAME, TEMPLATE_LOCATION) - .context(AddVolumeMountSnafu)?; + K8sContainer { + name: self.main_container.to_string(), + image: Some(self.cluster.image.image.clone()), + image_pull_policy: Some(self.cluster.image.image_pull_policy.clone()), + command: Some(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]), + args: Some(vec![self.pod_data.airflow_commands.join("\n")]), + env: Some(self.pod_data.env_vars.clone()), + ports: if ports.is_empty() { None } else { Some(ports) }, + volume_mounts: Some(volume_mounts), + resources: Some(self.role_group_config.resources.clone().into()), + readiness_probe, + liveness_probe, + ..K8sContainer::default() + } } - // for roles with an http endpoint - if let Some(http_port) = airflow_role.get_http_port() { - let probe = Probe { - tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(http_port.into()), - ..TCPSocketAction::default() + fn build_metrics_container(&self) -> K8sContainer { + let args = [ + COMMON_BASH_TRAP_FUNCTIONS.to_string(), + "prepare_signal_handlers".to_string(), + "/stackable/statsd_exporter &".to_string(), + "wait_for_termination $!".to_string(), + ] + .join("\n"); + + K8sContainer { + name: "metrics".to_string(), + image: Some(self.cluster.image.image.clone()), + image_pull_policy: Some(self.cluster.image.image_pull_policy.clone()), + command: Some(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]), + args: Some(vec![args]), + ports: Some(vec![ContainerPort { + name: Some(METRICS_PORT_NAME.to_string()), + container_port: METRICS_PORT.into(), + ..ContainerPort::default() + }]), + resources: Some(ResourceRequirements { + requests: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity("100m".to_string())), + ("memory".to_string(), Quantity("64Mi".to_string())), + ])), + limits: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity("200m".to_string())), + ("memory".to_string(), Quantity("64Mi".to_string())), + ])), + ..ResourceRequirements::default() }), - initial_delay_seconds: Some(60), - period_seconds: Some(10), - failure_threshold: Some(6), - ..Probe::default() - }; - airflow_container.readiness_probe(probe.clone()); - airflow_container.liveness_probe(probe); - airflow_container.add_container_port(HTTP_PORT_NAME, http_port.into()); + ..K8sContainer::default() + } } - let mut pvcs: Option> = None; + fn build_volumes(&self) -> Vec { + // REVIEW: controller_commons::create_volumes is called with the new validated type + // (&ValidatedContainerLogConfigChoice) instead of the old Option<&ContainerLogConfig>. + // This is safe because the logging config has already been validated during the + // validate stage. + let mut volumes = controller_commons::create_volumes( + &self.rolegroup_ref.object_name(), + &self.role_group_config.logging.airflow_container, + ); - if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) { - // Listener endpoints for the Webserver role will use persistent volumes - // so that load balancers can hard-code the target addresses. This will - // be the case even when no class is set (and the value defaults to - // cluster-internal) as the address should still be consistent. - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(listener_group_name), - &unversioned_recommended_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerVolumeSnafu)?; - pvcs = Some(vec![pvc]); + volumes.extend(self.pod_data.extra_volumes.clone()); + volumes.extend(self.pod_data.auth_volumes.clone()); + volumes.extend(self.pod_data.git_sync_volumes.clone()); + + if let Some(template_cm_name) = &self.pod_data.executor_template_configmap_name { + volumes.push(Volume { + name: TEMPLATE_VOLUME_NAME.to_string(), + config_map: Some( + stackable_operator::k8s_openapi::api::core::v1::ConfigMapVolumeSource { + name: template_cm_name.clone(), + ..Default::default() + }, + ), + ..Volume::default() + }); + } - airflow_container - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; + volumes } - // If the DAG is modularized we may encounter a timing issue whereby the celery worker - // has started *before* all modules referenced by the DAG have been fetched by gitsync - // and registered. This will result in ModuleNotFoundError errors. This can be avoided - // by running a one-off git-sync process in an init-container so that all DAG - // dependencies are fully loaded. The sidecar git-sync is then used for regular updates. - let use_git_sync_init_containers = matches!(executor, AirflowExecutor::CeleryExecutors { .. }); - add_git_sync_resources( - &mut pb, - &mut airflow_container, - git_sync_resources, - true, - use_git_sync_init_containers, - )?; + fn common_metadata(&self, resource_name: impl Into) -> ObjectMetaBuilder { + let mut builder = ObjectMetaBuilder::new(); - metadata_database_connection_details.add_to_container(&mut airflow_container); - if let Some((celery_result_backend, celery_broker)) = celery_database_connection_details { - celery_result_backend.add_to_container(&mut airflow_container); - celery_broker.add_to_container(&mut airflow_container); - } + builder + .name(resource_name) + .namespace(&self.cluster.namespace) + .ownerreference(ownerreference_from_resource(self.cluster, None, Some(true))) + .with_labels(self.recommended_labels()); - pb.add_container(airflow_container.build()); - - let metrics_container = ContainerBuilder::new("metrics") - .context(InvalidContainerNameSnafu)? - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![ - [ - COMMON_BASH_TRAP_FUNCTIONS.to_string(), - "prepare_signal_handlers".to_string(), - "/stackable/statsd_exporter &".to_string(), - "wait_for_termination $!".to_string(), - ] - .join("\n"), - ]) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("64Mi") - .with_memory_limit("64Mi") - .build(), - ) - .build(); - pb.add_container(metrics_container); - - pb.add_volumes(airflow.volumes().clone()) - .context(AddVolumeSnafu)?; - pb.add_volumes(controller_commons::create_volumes( - &rolegroup_ref.object_name(), - merged_airflow_config - .logging - .containers - .get(&Container::Airflow), - )) - .context(AddVolumeSnafu)?; + builder + } - if let AirflowExecutor::KubernetesExecutors { .. } = executor { - pb.add_volume( - VolumeBuilder::new(TEMPLATE_VOLUME_NAME) - .with_config_map(airflow.executor_template_configmap_name()) - .build(), + fn recommended_labels(&self) -> Labels { + use crate::framework::types::operator::*; + framework::kvp::label::recommended_labels( + self.cluster, + &ProductName::from_str_unsafe(APP_NAME), + &ProductVersion::from_str_unsafe( + &self.cluster.image.app_version_label_value.to_string(), + ), + &OperatorName::from_str_unsafe(OPERATOR_NAME), + &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), + &RoleName::from_str_unsafe(&self.rolegroup_ref.role), + &RoleGroupName::from_str_unsafe(&self.rolegroup_ref.role_group), ) - .context(AddVolumeSnafu)?; } +} - if merged_airflow_config.logging.enable_vector_agent { - match &airflow - .spec - .cluster_config - .vector_aggregator_config_map_name - { - Some(vector_aggregator_config_map_name) => { - pb.add_container(build_logging_container( - resolved_product_image, - merged_airflow_config - .logging - .containers - .get(&Container::Vector), - vector_aggregator_config_map_name, - )?); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } +// --------------------------------------------------------------------------- +// Apply +// --------------------------------------------------------------------------- + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum ApplyError { + #[snafu(display("failed to apply resource"))] + ApplyResource { + source: stackable_operator::cluster_resources::Error, + }, + + #[snafu(display("failed to delete orphaned resources"))] + DeleteOrphanedResources { + source: stackable_operator::cluster_resources::Error, + }, +} + +struct Applier<'a> { + client: &'a stackable_operator::client::Client, + cluster_resources: ClusterResources<'a>, +} + +impl<'a> Applier<'a> { + fn new( + client: &'a stackable_operator::client::Client, + cluster_resources: ClusterResources<'a>, + ) -> Self { + Applier { + client, + cluster_resources, } } - let mut pod_template = pb.build_template(); - pod_template.merge_from(role.config.pod_overrides.clone()); - if let Some(rolegroup) = rolegroup { - pod_template.merge_from(rolegroup.config.pod_overrides.clone()); - } - let restarter_label = - Label::try_from(("restarter.stackable.tech/enabled", "true")).context(BuildLabelSnafu)?; - - let metadata = build_rolegroup_metadata( - airflow, - &resolved_product_image, - &rolegroup_ref, - restarter_label, - rolegroup_ref.object_name(), - )?; + async fn apply( + mut self, + resources: KubernetesResources, + ) -> std::result::Result, ApplyError> { + let config_maps = self.add_resources(resources.config_maps).await?; + let service_accounts = self.add_resources(resources.service_accounts).await?; + let services = self.add_resources(resources.services).await?; + let role_bindings = self.add_resources(resources.role_bindings).await?; + let listeners = self.add_resources(resources.listeners).await?; + let stateful_sets = self.add_resources(resources.stateful_sets).await?; + let pod_disruption_budgets = self.add_resources(resources.pod_disruption_budgets).await?; + + self.cluster_resources + .delete_orphaned_resources(self.client) + .await + .context(apply_err::DeleteOrphanedResourcesSnafu)?; + + Ok(KubernetesResources { + stateful_sets, + config_maps, + services, + service_accounts, + role_bindings, + pod_disruption_budgets, + listeners, + _status: PhantomData, + }) + } - let statefulset_match_labels = Labels::role_group_selector( - airflow, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(BuildLabelSnafu)?; - - let statefulset_spec = StatefulSetSpec { - pod_management_policy: Some( - match airflow_role { - AirflowRole::Scheduler => { - "OrderedReady" // Scheduler pods should start after another, since part of their startup phase is initializing the database, see crd/src/lib.rs - } - AirflowRole::Webserver - | AirflowRole::Worker - | AirflowRole::DagProcessor - | AirflowRole::Triggerer => "Parallel", - } - .to_string(), - ), - replicas: rolegroup.and_then(|rg| rg.replicas).map(i32::from), - selector: LabelSelector { - match_labels: Some(statefulset_match_labels.into()), - ..LabelSelector::default() - }, - service_name: stateful_set_service_name(rolegroup_ref), - template: pod_template, - volume_claim_templates: pvcs, - ..StatefulSetSpec::default() - }; + async fn add_resources( + &mut self, + resources: Vec, + ) -> std::result::Result, ApplyError> { + let mut applied = vec![]; + for resource in resources { + let applied_resource = self + .cluster_resources + .add(self.client, resource) + .await + .context(apply_err::ApplyResourceSnafu)?; + applied.push(applied_resource); + } + Ok(applied) + } +} - Ok(StatefulSet { - metadata, - spec: Some(statefulset_spec), - status: None, - }) +/// Module-like namespace for Snafu context selectors for ApplyError. +mod apply_err { + pub(super) use super::{ApplyResourceSnafu, DeleteOrphanedResourcesSnafu}; } -fn build_logging_container( - resolved_product_image: &ResolvedProductImage, - log_config: Option<&ContainerLogConfig>, - vector_aggregator_config_map_name: &str, -) -> Result { - product_logging::framework::vector_container( - resolved_product_image, - CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - log_config, - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu) +// --------------------------------------------------------------------------- +// Update status +// --------------------------------------------------------------------------- + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum UpdateStatusError { + #[snafu(display("failed to update status"))] + PatchStatus { + source: stackable_operator::client::Error, + }, } -#[allow(clippy::too_many_arguments)] -fn build_executor_template_config_map( +async fn update_status( + client: &stackable_operator::client::Client, airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - sa_name: &str, - merged_executor_config: &ExecutorConfig, - env_overrides: &HashMap, - pod_overrides: &PodTemplateSpec, - rolegroup_ref: &RoleGroupRef, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, -) -> Result { - let mut pb = PodBuilder::new(); - let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&build_recommended_labels( + applied_resources: KubernetesResources, +) -> std::result::Result<(), UpdateStatusError> { + let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + for stateful_set in applied_resources.stateful_sets { + ss_cond_builder.add(stateful_set); + } + + let cluster_operation_cond_builder = + ClusterOperationsConditionBuilder::new(&airflow.spec.cluster_operation); + + let status = AirflowClusterStatus { + conditions: compute_conditions( airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - "executor", - "executor-template", - )) - .context(ObjectMetaSnafu)? - .build(); + &[&ss_cond_builder, &cluster_operation_cond_builder], + ), + }; - pb.metadata(pb_metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .affinity(&merged_executor_config.affinity) - .service_account_name(sa_name) - .restart_policy("Never") - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + client + .apply_patch_status(OPERATOR_NAME, airflow, &status) + .await + .context(PatchStatusSnafu)?; - add_executor_graceful_shutdown_config(merged_executor_config, &mut pb) - .context(GracefulShutdownSnafu)?; + Ok(()) +} - // N.B. this "base" name is an airflow requirement and should not be changed! - // See https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.4.0/kubernetes_executor.html#base-image - let mut airflow_container = - ContainerBuilder::new(&Container::Base.to_string()).context(InvalidContainerNameSnafu)?; +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- - // Works too, had been changed - add_authentication_volumes_and_volume_mounts( - authentication_config, - &mut airflow_container, - &mut pb, - )?; - airflow_container - .image_from_product_image(resolved_product_image) - .resources(merged_executor_config.resources.clone().into()) - .add_env_vars(build_airflow_template_envs( - airflow, - env_overrides, - merged_executor_config.logging.enable_vector_agent, - metadata_database_connection_details, - git_sync_resources, - resolved_product_image, - )) - .add_volume_mounts(airflow.volume_mounts()) - .context(AddVolumeMountSnafu)? - .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)?; - - add_git_sync_resources( - &mut pb, - &mut airflow_container, - git_sync_resources, - false, - true, - )?; +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; - metadata_database_connection_details.add_to_container(&mut airflow_container); - - pb.add_container(airflow_container.build()); - pb.add_volumes(airflow.volumes().clone()) - .context(AddVolumeSnafu)?; - pb.add_volumes(controller_commons::create_volumes( - &rolegroup_ref.object_name(), - merged_executor_config - .logging - .containers - .get(&Container::Airflow), - )) - .context(AddVolumeSnafu)?; + use stackable_operator::{ + commons::{ + affinity::StackableAffinity, + product_image_selection::ResolvedProductImage, + resources::{NoRuntimeLimits, Resources}, + }, + k8s_openapi::api::core::v1::PodTemplateSpec, + kube::Resource, + kvp::LabelValue, + product_logging::spec::{ + AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging, + }, + shared::time::Duration, + }; - if merged_executor_config.logging.enable_vector_agent { - match &airflow - .spec - .cluster_config - .vector_aggregator_config_map_name - { - Some(vector_aggregator_config_map_name) => { - pb.add_container(build_logging_container( - resolved_product_image, - merged_executor_config - .logging - .containers - .get(&Container::Vector), - vector_aggregator_config_map_name, - )?); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } + use super::*; + use crate::crd::{AirflowConfig, AirflowStorageConfig, Container}; + + // --- validate tests --- + + fn airflow_config_with_logging(enable_vector: bool) -> AirflowConfig { + let mut containers = BTreeMap::new(); + containers.insert( + Container::Airflow, + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + ); + if enable_vector { + containers.insert( + Container::Vector, + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + ); + } + AirflowConfig { + resources: Default::default(), + logging: Logging { + enable_vector_agent: enable_vector, + containers, + }, + affinity: Default::default(), + graceful_shutdown_timeout: Some(Duration::from_secs(120)), } } - let mut pod_template = pb.build_template(); - pod_template.merge_from(pod_overrides.clone()); - - let mut cm_builder = ConfigMapBuilder::new(); - - let restarter_label = - Label::try_from(("restarter.stackable.tech/enabled", "true")).context(BuildLabelSnafu)?; - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(airflow.executor_template_configmap_name()) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - "executor", - "executor-template", - )) - .context(ObjectMetaSnafu)? - .with_label(restarter_label) - .build(), - ) - .add_data( - TEMPLATE_NAME, - serde_yaml::to_string(&pod_template).context(PodTemplateSerdeSnafu)?, + #[test] + fn test_validate_airflow_config_without_vector() { + let config = airflow_config_with_logging(false); + let result = validate_airflow_config(&config, None, String::new()); + assert!(result.is_ok()); + let validated = result.unwrap(); + assert!(validated.logging.vector_container.is_none()); + assert!(!validated.logging.is_vector_agent_enabled()); + assert_eq!( + validated.graceful_shutdown_timeout, + Duration::from_secs(120) ); + } - cm_builder.build().context(PodTemplateConfigMapSnafu) -} + #[test] + fn test_validate_airflow_config_with_vector() { + let config = airflow_config_with_logging(true); + let result = + validate_airflow_config(&config, Some("vector-aggregator-discovery"), String::new()); + assert!(result.is_ok()); + let validated = result.unwrap(); + assert!(validated.logging.vector_container.is_some()); + assert!(validated.logging.is_vector_agent_enabled()); + } -pub fn error_policy( - _obj: Arc>, - error: &Error, - _ctx: Arc, -) -> Action { - match error { - // root object is invalid, will be requeued when modified anyway - Error::InvalidAirflowCluster { .. } => Action::await_change(), + #[test] + fn test_validate_vector_enabled_missing_config_map() { + let config = airflow_config_with_logging(true); + let result = validate_airflow_config(&config, None, String::new()); + assert!(result.is_err()); + } - _ => Action::requeue(*Duration::from_secs(10)), + #[test] + fn test_validate_missing_graceful_shutdown() { + let mut config = airflow_config_with_logging(false); + config.graceful_shutdown_timeout = None; + let result = validate_airflow_config(&config, None, String::new()); + assert!(result.is_err()); } -} -fn add_authentication_volumes_and_volume_mounts( - authentication_config: &AirflowClientAuthenticationDetailsResolved, - cb: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // Different authentication entries can reference the same secret - // class or TLS certificate. It must be ensured that the volumes - // and volume mounts are only added once in such a case. + #[test] + fn test_validate_ok() { + let (airflow, image) = test_objects(); + let result = validate_and_assemble( + &airflow, + &image, + BTreeMap::new(), + BTreeMap::new(), + vec![], + BTreeMap::new(), + ); + assert!(result.is_ok()); + let validated = result.unwrap(); + assert_eq!(validated.name.to_string(), "my-airflow"); + assert_eq!(validated.namespace.to_string(), "default"); + } - let mut ldap_authentication_providers = BTreeSet::new(); - let mut tls_client_credentials = BTreeSet::new(); + #[test] + fn test_validate_err_missing_name() { + test_validate_err( + |airflow, _| airflow.metadata.name = None, + ValidateErrorDiscriminants::InvalidClusterName, + ); + } - for auth_class_resolved in &authentication_config.authentication_classes_resolved { - match auth_class_resolved { - AirflowAuthenticationClassResolved::Ldap { provider } => { - ldap_authentication_providers.insert(provider); - } - AirflowAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - } - } + #[test] + fn test_validate_err_missing_namespace() { + test_validate_err( + |airflow, _| airflow.metadata.namespace = None, + ValidateErrorDiscriminants::ObjectHasNoNamespace, + ); } - for provider in ldap_authentication_providers { - provider - .add_volumes_and_mounts(pb, vec![cb]) - .context(AddLdapVolumesAndVolumeMountsSnafu)?; + #[test] + fn test_validate_err_missing_uid() { + test_validate_err( + |airflow, _| airflow.metadata.uid = None, + ValidateErrorDiscriminants::ObjectHasNoUid, + ); } - for tls in tls_client_credentials { - tls.add_volumes_and_mounts(pb, vec![cb]) - .context(AddTlsVolumesAndVolumeMountsSnafu)?; + #[test] + fn test_validate_err_invalid_cluster_name() { + test_validate_err( + |airflow, _| { + airflow.metadata.name = + Some("THIS-IS-NOT-A-VALID-DNS-LABEL-NAME-BECAUSE-UPPERCASE".to_string()) + }, + ValidateErrorDiscriminants::InvalidClusterName, + ); } - Ok(()) -} -fn add_git_sync_resources( - pb: &mut PodBuilder, - cb: &mut ContainerBuilder, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, - add_sidecar_containers: bool, - add_init_containers: bool, -) -> Result<()> { - if add_sidecar_containers { - for container in git_sync_resources.git_sync_containers.iter().cloned() { - pb.add_container(container); - } + #[test] + fn test_validate_err_invalid_namespace() { + test_validate_err( + |airflow, _| airflow.metadata.namespace = Some("INVALID NAMESPACE".to_string()), + ValidateErrorDiscriminants::InvalidClusterNamespace, + ); } - if add_init_containers { - for container in git_sync_resources.git_sync_init_containers.iter().cloned() { - pb.add_init_container(container); + + #[test] + fn test_validate_err_invalid_uid() { + test_validate_err( + |airflow, _| airflow.metadata.uid = Some("not-a-uuid".to_string()), + ValidateErrorDiscriminants::InvalidClusterUid, + ); + } + + fn test_validate_err( + mutate: fn(&mut v1alpha2::AirflowCluster, &mut ResolvedProductImage), + expected: ValidateErrorDiscriminants, + ) { + let (mut airflow, mut image) = test_objects(); + mutate(&mut airflow, &mut image); + let result = validate_and_assemble( + &airflow, + &image, + BTreeMap::new(), + BTreeMap::new(), + vec![], + BTreeMap::new(), + ); + match result { + Err(err) => assert_eq!(expected, ValidateErrorDiscriminants::from(err)), + Ok(_) => panic!("validate should have failed with {expected:?}"), } } - pb.add_volumes(git_sync_resources.git_content_volumes.to_owned()) - .context(AddVolumeSnafu)?; - pb.add_volumes(git_sync_resources.git_ssh_volumes.to_owned()) - .context(AddVolumeSnafu)?; - pb.add_volumes(git_sync_resources.git_ca_cert_volumes.to_owned()) - .context(AddVolumeSnafu)?; - cb.add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) - .context(AddVolumeMountSnafu)?; - Ok(()) + fn test_objects() -> (v1alpha2::AirflowCluster, ResolvedProductImage) { + let airflow = v1alpha2::AirflowCluster { + metadata: ObjectMeta { + name: Some("my-airflow".to_string()), + namespace: Some("default".to_string()), + uid: Some("e6ac237d-a6d4-43a1-8135-f36506110912".to_string()), + ..ObjectMeta::default() + }, + spec: serde_json::from_value(serde_json::json!({ + "image": { "productVersion": "2.10.4" }, + "clusterConfig": { + "credentialsSecretName": "airflow-admin-credentials", + "metadataDatabase": { + "postgresql": { + "host": "airflow-postgresql", + "database": "airflow", + "credentialsSecretName": "airflow-postgresql-credentials" + } + } + }, + "kubernetesExecutors": { "config": {} }, + "webservers": { "roleGroups": { "default": { "config": {} } } }, + "schedulers": { "roleGroups": { "default": { "config": {} } } } + })) + .expect("test spec JSON should be valid"), + status: None, + }; + + let image = ResolvedProductImage { + product_version: "2.10.4".to_owned(), + app_version_label_value: LabelValue::from_str("2.10.4-stackable0.0.0-dev") + .expect("valid label value"), + image: "oci.stackable.tech/sdp/airflow:2.10.4-stackable0.0.0-dev".to_string(), + image_pull_policy: "Always".to_owned(), + pull_secrets: None, + }; + + (airflow, image) + } + + // --- build tests --- + + #[test] + fn test_build() { + let validated = validated_cluster(); + + let resources = build(&validated); + + assert_eq!( + vec![ + "my-airflow-scheduler-default", + "my-airflow-webserver-default", + ], + extract_resource_names(&resources.stateful_sets) + ); + assert_eq!( + vec![ + "my-airflow-scheduler-default-headless", + "my-airflow-scheduler-default-metrics", + "my-airflow-webserver-default-headless", + "my-airflow-webserver-default-metrics", + ], + extract_resource_names(&resources.services) + ); + assert_eq!( + vec![ + "my-airflow-scheduler-default", + "my-airflow-webserver-default", + ], + extract_resource_names(&resources.config_maps) + ); + assert_eq!( + vec!["my-airflow-serviceaccount"], + extract_resource_names(&resources.service_accounts) + ); + assert_eq!( + vec!["my-airflow-rolebinding"], + extract_resource_names(&resources.role_bindings) + ); + assert_eq!( + vec!["my-airflow-scheduler", "my-airflow-webserver"], + extract_resource_names(&resources.pod_disruption_budgets) + ); + assert_eq!( + vec!["my-airflow-webserver"], + extract_resource_names(&resources.listeners) + ); + } + + fn extract_resource_names(resources: &[impl Resource]) -> Vec<&str> { + let mut names: Vec<&str> = resources + .iter() + .filter_map(|r| r.meta().name.as_ref()) + .map(|n| n.as_str()) + .collect(); + names.sort(); + names + } + + fn validated_cluster() -> ValidatedAirflowCluster { + let image = ResolvedProductImage { + product_version: "2.10.4".to_owned(), + app_version_label_value: LabelValue::from_str("2.10.4-stackable0.0.0-dev") + .expect("valid label value"), + image: "oci.stackable.tech/sdp/airflow:2.10.4-stackable0.0.0-dev".to_string(), + image_pull_policy: "Always".to_owned(), + pull_secrets: None, + }; + + let logging = ValidatedLogging { + airflow_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + git_sync_container_log_config: ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + }; + + let role_group_config = ValidatedRoleGroupConfig { + resources: Resources::::default(), + logging: logging.clone(), + affinity: StackableAffinity::default(), + graceful_shutdown_timeout: Duration::from_secs(120), + config_file_content: String::new(), + }; + + let pod_data = PrecomputedPodData { + env_vars: vec![], + airflow_commands: vec!["airflow webserver".to_string()], + auth_volumes: vec![], + auth_volume_mounts: vec![], + extra_volumes: vec![], + extra_volume_mounts: vec![], + git_sync_containers: vec![], + git_sync_init_containers: vec![], + git_sync_volumes: vec![], + git_sync_volume_mounts: vec![], + vector_container: None, + service_account_name: "my-airflow-serviceaccount".to_string(), + replicas: Some(1), + pod_overrides: PodTemplateSpec::default(), + executor: AirflowExecutor::KubernetesExecutors { + common_configuration: Box::default(), + }, + executor_template_configmap_name: None, + listener_volume_claim_template: None, + }; + + let role_groups = BTreeMap::from([ + ( + AirflowRole::Webserver, + BTreeMap::from([("default".to_string(), role_group_config.clone())]), + ), + ( + AirflowRole::Scheduler, + BTreeMap::from([("default".to_string(), role_group_config)]), + ), + ]); + + let precomputed_pod_data = BTreeMap::from([ + ( + AirflowRole::Webserver, + BTreeMap::from([("default".to_string(), pod_data.clone())]), + ), + ( + AirflowRole::Scheduler, + BTreeMap::from([("default".to_string(), pod_data)]), + ), + ]); + + // Role configs: PDB enabled for both roles; Webserver also gets a listener + let role_configs = BTreeMap::from([ + ( + AirflowRole::Scheduler, + ValidatedRoleConfig { + pdb_enabled: true, + pdb_max_unavailable: None, + listener_class: None, + group_listener_name: None, + }, + ), + ( + AirflowRole::Webserver, + ValidatedRoleConfig { + pdb_enabled: true, + pdb_max_unavailable: None, + listener_class: Some("cluster-internal".to_string()), + group_listener_name: Some("my-airflow-webserver".to_string()), + }, + ), + ]); + + ValidatedAirflowCluster::new( + image, + ClusterName::from_str_unsafe("my-airflow"), + NamespaceName::from_str_unsafe("default"), + Uid::from_str_unsafe("e6ac237d-a6d4-43a1-8135-f36506110912"), + role_groups, + precomputed_pod_data, + vec![], + role_configs, + AirflowExecutor::KubernetesExecutors { + common_configuration: Box::default(), + }, + ) + } } diff --git a/rust/operator-binary/src/controller_commons.rs b/rust/operator-binary/src/controller_commons.rs index 7d16c41b..9cac0007 100644 --- a/rust/operator-binary/src/controller_commons.rs +++ b/rust/operator-binary/src/controller_commons.rs @@ -1,24 +1,23 @@ use stackable_operator::{ builder::pod::volume::VolumeBuilder, k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, - product_logging::{ - self, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, + product_logging, }; -use crate::crd::MAX_LOG_FILES_SIZE; +use crate::{ + crd::MAX_LOG_FILES_SIZE, + framework::product_logging::framework::ValidatedContainerLogConfigChoice, +}; pub const CONFIG_VOLUME_NAME: &str = "config"; pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; pub const LOG_VOLUME_NAME: &str = "log"; +// REVIEW: parameter changed from Option<&ContainerLogConfig> to &ValidatedContainerLogConfigChoice. +// Pattern matching is simpler now because we match on a flat enum instead of nested Option. pub fn create_volumes( config_map_name: &str, - log_config: Option<&ContainerLogConfig>, + log_config: &ValidatedContainerLogConfigChoice, ) -> Vec { let mut volumes = Vec::new(); @@ -38,17 +37,11 @@ pub fn create_volumes( ..Volume::default() }); - if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = log_config - { + if let ValidatedContainerLogConfigChoice::Custom(custom_config_map) = log_config { volumes.push(Volume { name: LOG_CONFIG_VOLUME_NAME.into(), config_map: Some(ConfigMapVolumeSource { - name: config_map.into(), + name: custom_config_map.as_ref().into(), ..ConfigMapVolumeSource::default() }), ..Volume::default() diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index eb2ebc1c..6fdcc9fd 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -178,7 +178,8 @@ async fn main() -> anyhow::Result<()> { ) .graceful_shutdown_on(sigterm_watcher.handle()) .run( - airflow_controller::reconcile_airflow, + // REVIEW: renamed from reconcile_airflow to reconcile + airflow_controller::reconcile, airflow_controller::error_policy, Arc::new(airflow_controller::Ctx { client: client.clone(), diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 51572729..0f56d2af 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,59 +1,37 @@ use std::fmt::{Display, Write}; -use snafu::Snafu; use stackable_operator::{ builder::configmap::ConfigMapBuilder, commons::product_image_selection::ResolvedProductImage, kube::Resource, - product_logging::{ - self, - spec::{ - AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging, - }, - }, + product_logging::{self, spec::AutomaticContainerLogConfig}, role_utils::RoleGroupRef, }; -use crate::crd::STACKABLE_LOG_DIR; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to retrieve the ConfigMap [{cm_name}]"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - #[snafu(display("failed to retrieve the entry [{entry}] for ConfigMap [{cm_name}]"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, +use crate::{ + crd::STACKABLE_LOG_DIR, + framework::product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, }, - #[snafu(display("vectorAggregatorConfigMapName must be set"))] - MissingVectorAggregatorAddress, -} - -type Result = std::result::Result; +}; const LOG_CONFIG_FILE: &str = "log_config.py"; const LOG_FILE: &str = "airflow.py.json"; +// REVIEW: signature changed from Logging + two container keys to pre-validated types. +// The function is now infallible — validation happens earlier in the pipeline. /// Extend the ConfigMap with logging and Vector configurations -pub fn extend_config_map_with_log_config( +pub fn extend_config_map_with_log_config( rolegroup: &RoleGroupRef, - logging: &Logging, - main_container: &C, - vector_container: &C, + main_container: &impl Display, + main_container_log_config: &ValidatedContainerLogConfigChoice, + vector_config: Option<&VectorContainerLogConfig>, cm_builder: &mut ConfigMapBuilder, resolved_product_image: &ResolvedProductImage, -) -> Result<()> -where - C: Clone + Ord + Display, +) where K: Resource, { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(main_container) - { + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = main_container_log_config { let log_dir = format!("{STACKABLE_LOG_DIR}/{main_container}"); cm_builder.add_data( LOG_CONFIG_FILE, @@ -61,23 +39,20 @@ where ); } - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(vector_container) - { - Some(log_config) - } else { - None - }; + if let Some(vector_config) = vector_config { + let vector_log_config = if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &vector_config.log_config + { + Some(log_config) + } else { + None + }; - if logging.enable_vector_agent { cm_builder.add_data( product_logging::framework::VECTOR_CONFIG_FILE, product_logging::framework::create_vector_config(rolegroup, vector_log_config), ); } - - Ok(()) } fn create_airflow_config( From 0caca3868ac019d0cd4e1f3b5025df8bb7bcabdc Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 11:09:58 +0200 Subject: [PATCH 11/14] refactor: remove dead code from operations/, service.rs, and crd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete the operations/ module (graceful_shutdown, pdb) — its functionality is now handled inline by the validate and build stages. Remove dead functions from service.rs (service building is now inline in the controller build stage) and crd/mod.rs (build_recommended_labels replaced by framework::kvp::label functions). Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/crd/mod.rs | 19 --- rust/operator-binary/src/main.rs | 1 - .../src/operations/graceful_shutdown.rs | 42 ----- rust/operator-binary/src/operations/mod.rs | 2 - rust/operator-binary/src/operations/pdb.rs | 89 ---------- rust/operator-binary/src/service.rs | 153 +----------------- 6 files changed, 1 insertion(+), 305 deletions(-) delete mode 100644 rust/operator-binary/src/operations/graceful_shutdown.rs delete mode 100644 rust/operator-binary/src/operations/mod.rs delete mode 100644 rust/operator-binary/src/operations/pdb.rs diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 2354b174..3b48d99a 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -27,7 +27,6 @@ use stackable_operator::{ apimachinery::pkg::api::resource::Quantity, }, kube::{CustomResource, ResourceExt}, - kvp::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{self, Configuration}, product_logging::{ @@ -1095,24 +1094,6 @@ fn default_resources(role: &AirflowRole) -> ResourcesFragment( - owner: &'a T, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -} #[cfg(test)] mod tests { diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 6fdcc9fd..8bc2b586 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -44,7 +44,6 @@ mod controller_commons; mod crd; mod env_vars; mod framework; -mod operations; mod product_logging; mod service; mod util; diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/operations/graceful_shutdown.rs deleted file mode 100644 index dde8e074..00000000 --- a/rust/operator-binary/src/operations/graceful_shutdown.rs +++ /dev/null @@ -1,42 +0,0 @@ -use snafu::{ResultExt, Snafu}; -use stackable_operator::builder::pod::PodBuilder; - -use crate::crd::{AirflowConfig, ExecutorConfig}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("Failed to set terminationGracePeriod"))] - SetTerminationGracePeriod { - source: stackable_operator::builder::pod::Error, - }, -} - -pub fn add_airflow_graceful_shutdown_config( - merged_config: &AirflowConfig, - pod_builder: &mut PodBuilder, -) -> Result<(), Error> { - // This must be always set by the merge mechanism, as we provide a default value, - // users can not disable graceful shutdown. - if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { - pod_builder - .termination_grace_period(&graceful_shutdown_timeout) - .context(SetTerminationGracePeriodSnafu)?; - } - - Ok(()) -} - -pub fn add_executor_graceful_shutdown_config( - merged_config: &ExecutorConfig, - pod_builder: &mut PodBuilder, -) -> Result<(), Error> { - // This must be always set by the merge mechanism, as we provide a default value, - // users can not disable graceful shutdown. - if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { - pod_builder - .termination_grace_period(&graceful_shutdown_timeout) - .context(SetTerminationGracePeriodSnafu)?; - } - - Ok(()) -} diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs deleted file mode 100644 index 92ca2ec7..00000000 --- a/rust/operator-binary/src/operations/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod graceful_shutdown; -pub mod pdb; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs deleted file mode 100644 index b3261678..00000000 --- a/rust/operator-binary/src/operations/pdb.rs +++ /dev/null @@ -1,89 +0,0 @@ -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, - commons::pdb::PdbConfig, kube::ResourceExt, -}; - -use crate::{ - airflow_controller::AIRFLOW_CONTROLLER_NAME, - crd::{APP_NAME, AirflowExecutor, AirflowRole, OPERATOR_NAME, v1alpha2}, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] - CreatePdb { - source: stackable_operator::builder::pdb::Error, - role: String, - }, - #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - name: String, - }, -} - -pub async fn add_pdbs( - pdb: &PdbConfig, - airflow: &v1alpha2::AirflowCluster, - role: &AirflowRole, - client: &Client, - cluster_resources: &mut ClusterResources<'_>, -) -> Result<(), Error> { - if !pdb.enabled { - return Ok(()); - } - - let max_unavailable = pdb.max_unavailable.unwrap_or(match role { - AirflowRole::Scheduler => max_unavailable_schedulers(), - AirflowRole::Webserver => max_unavailable_webservers(), - AirflowRole::DagProcessor => max_unavailable_dag_processors(), - AirflowRole::Triggerer => max_unavailable_triggerers(), - AirflowRole::Worker => match airflow.spec.executor { - AirflowExecutor::CeleryExecutors { .. } => max_unavailable_workers(), - AirflowExecutor::KubernetesExecutors { .. } => { - // In case Airflow creates the Pods, we don't want to influence that. - return Ok(()); - } - }, - }); - let pdb = PodDisruptionBudgetBuilder::new_with_role( - airflow, - APP_NAME, - &role.to_string(), - OPERATOR_NAME, - AIRFLOW_CONTROLLER_NAME, - ) - .with_context(|_| CreatePdbSnafu { - role: role.to_string(), - })? - .with_max_unavailable(max_unavailable) - .build(); - let pdb_name = pdb.name_any(); - cluster_resources - .add(client, pdb) - .await - .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; - - Ok(()) -} - -fn max_unavailable_schedulers() -> u16 { - 1 -} - -fn max_unavailable_workers() -> u16 { - 1 -} - -fn max_unavailable_webservers() -> u16 { - 1 -} - -fn max_unavailable_dag_processors() -> u16 { - 1 -} - -fn max_unavailable_triggerers() -> u16 { - 1 -} diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index 67df3cec..6b977032 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -1,112 +1,7 @@ -use std::collections::BTreeMap; +use stackable_operator::{kube::Resource, role_utils::RoleGroupRef}; -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, - kube::Resource, - kvp::{Annotations, Labels, ObjectLabels}, - role_utils::RoleGroupRef, -}; - -use crate::crd::{HTTP_PORT, HTTP_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha2}; - -pub const METRICS_SERVICE_SUFFIX: &str = "metrics"; pub const HEADLESS_SERVICE_SUFFIX: &str = "headless"; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, -} - -/// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup -/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -pub fn build_rolegroup_headless_service( - airflow: &v1alpha2::AirflowCluster, - rolegroup_ref: &RoleGroupRef, - object_labels: ObjectLabels, - selector: BTreeMap, -) -> Result { - let ports = headless_service_ports(); - - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(rolegroup_headless_service_name( - &rolegroup_ref.object_name(), - )) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(MetadataBuildSnafu)? - .build(); - - let service_spec = ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(ports), - selector: Some(selector), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }; - - Ok(Service { - metadata, - spec: Some(service_spec), - status: None, - }) -} - -/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. -pub fn build_rolegroup_metrics_service( - airflow: &v1alpha2::AirflowCluster, - rolegroup_ref: &RoleGroupRef, - object_labels: ObjectLabels, - selector: BTreeMap, -) -> Result { - let ports = metrics_service_ports(); - - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(rolegroup_metrics_service_name(&rolegroup_ref.object_name())) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(MetadataBuildSnafu)? - .with_labels(prometheus_labels()) - .with_annotations(prometheus_annotations()) - .build(); - - let service_spec = ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(ports), - selector: Some(selector), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }; - - Ok(Service { - metadata, - spec: Some(service_spec), - status: None, - }) -} - // REVIEW: made generic so the new controller can call this with // RoleGroupRef instead of RoleGroupRef pub fn stateful_set_service_name( @@ -117,52 +12,6 @@ pub fn stateful_set_service_name( )) } -/// Returns the metrics rolegroup service name `---`. -// TODO: Replace by operator.rs functions -fn rolegroup_metrics_service_name(role_group_ref_object_name: &str) -> String { - format!("{role_group_ref_object_name}-{METRICS_SERVICE_SUFFIX}") -} - -/// Returns the headless rolegroup service name `---`. -// TODO: Replace by operator.rs functions fn rolegroup_headless_service_name(role_group_ref_object_name: &str) -> String { format!("{role_group_ref_object_name}-{HEADLESS_SERVICE_SUFFIX}") } - -fn headless_service_ports() -> Vec { - vec![ServicePort { - name: Some(HTTP_PORT_NAME.to_string()), - port: HTTP_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] -} - -fn metrics_service_ports() -> Vec { - vec![ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] -} - -/// Common labels for Prometheus -fn prometheus_labels() -> Labels { - Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") -} - -/// Common annotations for Prometheus -/// -/// These annotations can be used in a ServiceMonitor. -/// -/// see also -fn prometheus_annotations() -> Annotations { - Annotations::try_from([ - ("prometheus.io/path".to_owned(), "/metrics".to_owned()), - ("prometheus.io/port".to_owned(), METRICS_PORT.to_string()), - ("prometheus.io/scheme".to_owned(), "http".to_owned()), - ("prometheus.io/scrape".to_owned(), "true".to_owned()), - ]) - .expect("should be valid annotations") -} From 8b97621c39ee9a1846700891f7136d08e16235d3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 11:22:33 +0200 Subject: [PATCH 12/14] refactor: extract controller into submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the monolithic airflow_controller.rs into controller/ with submodules: dereference, validate, build (+ build/role_group_builder), apply, and update_status. This is a purely structural refactor — no logic changes. Each pipeline stage now lives in its own file for easier navigation and review. Rename airflow_controller module to controller in main.rs. Co-Authored-By: Claude Opus 4.6 --- .../operator-binary/src/airflow_controller.rs | 2770 ----------------- rust/operator-binary/src/controller.rs | 378 +++ rust/operator-binary/src/controller/apply.rs | 83 + rust/operator-binary/src/controller/build.rs | 529 ++++ .../controller/build/role_group_builder.rs | 416 +++ .../src/controller/dereference.rs | 107 + .../src/controller/update_status.rs | 47 + .../src/controller/validate.rs | 1320 ++++++++ rust/operator-binary/src/main.rs | 10 +- 9 files changed, 2885 insertions(+), 2775 deletions(-) delete mode 100644 rust/operator-binary/src/airflow_controller.rs create mode 100644 rust/operator-binary/src/controller.rs create mode 100644 rust/operator-binary/src/controller/apply.rs create mode 100644 rust/operator-binary/src/controller/build.rs create mode 100644 rust/operator-binary/src/controller/build/role_group_builder.rs create mode 100644 rust/operator-binary/src/controller/dereference.rs create mode 100644 rust/operator-binary/src/controller/update_status.rs create mode 100644 rust/operator-binary/src/controller/validate.rs diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs deleted file mode 100644 index aea53f8f..00000000 --- a/rust/operator-binary/src/airflow_controller.rs +++ /dev/null @@ -1,2770 +0,0 @@ -//! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`] -//! -//! Pipeline architecture: dereference -> validate -> build -> apply -> update_status - -// --------------------------------------------------------------------------- -// Imports -// --------------------------------------------------------------------------- - -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - marker::PhantomData, - str::FromStr, - sync::Arc, -}; - -use const_format::concatcp; -use product_config::{ProductConfigManager, types::PropertyNameKind}; -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::{ - builder::{ - configmap::ConfigMapBuilder, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, - container::ContainerBuilder, - resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, - volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, - }, - }, - cli::OperatorEnvironmentOptions, - cluster_resources::{ClusterResource, ClusterResourceApplyStrategy, ClusterResources}, - commons::{ - affinity::StackableAffinity, - product_image_selection::ResolvedProductImage, - rbac::build_rbac_resources, - resources::{NoRuntimeLimits, Resources}, - }, - crd::{git_sync, listener}, - database_connections::drivers::{ - celery::CeleryDatabaseConnectionDetails, sqlalchemy::SqlAlchemyDatabaseConnectionDetails, - }, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ - Affinity, ConfigMap, Container as K8sContainer, ContainerPort, EnvVar, - PersistentVolumeClaim, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - ResourceRequirements, Service, ServiceAccount, ServicePort, ServiceSpec, - TCPSocketAction, Volume, VolumeMount, - }, - policy::v1::PodDisruptionBudget, - rbac::v1::RoleBinding, - }, - apimachinery::pkg::{ - api::resource::Quantity, apis::meta::v1::LabelSelector, util::intstr::IntOrString, - }, - }, - kube::{ - Resource, ResourceExt, - api::ObjectMeta, - core::{DeserializeGuard, error_boundary}, - runtime::{controller::Action, reflector::ObjectRef}, - }, - kvp::{Annotation, Annotations, Label, Labels, ObjectLabels}, - logging::controller::ReconcilerError, - product_config_utils::{ - env_vars_from, env_vars_from_rolegroup_config, transform_all_roles_to_config, - validate_all_roles_and_groups_config, - }, - product_logging::{self, framework::LoggingError, spec::Logging}, - role_utils::RoleGroupRef, - shared::time::Duration, - status::condition::{ - compute_conditions, operations::ClusterOperationsConditionBuilder, - statefulset::StatefulSetConditionBuilder, - }, - utils::COMMON_BASH_TRAP_FUNCTIONS, -}; -use stackable_operator::commons::random_secret_creation; -use stackable_operator::product_logging::spec::ContainerLogConfig; -use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; - -use crate::{ - crd::{ - AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowExecutor, - AirflowRole, AirflowStorageConfig, CONFIG_PATH, Container, ExecutorConfig, HTTP_PORT, - HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, - METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, - TEMPLATE_VOLUME_NAME, - authentication::{ - AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, - }, - authorization::AirflowAuthorizationResolved, - internal_secret::{FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY}, - v1alpha2, - }, - controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, - env_vars, - framework::{ - self, HasName, HasUid, NameIsValidLabelValue, - builder::meta::ownerreference_from_resource, - product_logging::framework::{ - ValidatedContainerLogConfigChoice, VectorContainerLogConfig, - validate_logging_configuration_for_container, - }, - types::{ - kubernetes::{ConfigMapName, NamespaceName, Uid}, - operator::ClusterName, - }, - }, - product_logging::extend_config_map_with_log_config, - service::stateful_set_service_name, -}; - -// --------------------------------------------------------------------------- -// Constants and context -// --------------------------------------------------------------------------- - -pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; -pub const CONTAINER_IMAGE_BASE_NAME: &str = "airflow"; -pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = - concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); - -pub struct Ctx { - pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, - pub operator_environment: OperatorEnvironmentOptions, -} - -// --------------------------------------------------------------------------- -// Validated types -// --------------------------------------------------------------------------- - -pub(crate) struct Prepared; -pub(crate) struct Applied; - -pub(crate) struct KubernetesResources { - pub stateful_sets: Vec, - pub config_maps: Vec, - pub services: Vec, - pub service_accounts: Vec, - pub role_bindings: Vec, - pub pod_disruption_budgets: Vec, - pub listeners: Vec, - pub _status: PhantomData, -} - -#[derive(Clone, Debug)] -pub struct ValidatedRoleConfig { - pub pdb_enabled: bool, - pub pdb_max_unavailable: Option, - pub listener_class: Option, - pub group_listener_name: Option, -} - -#[derive(Clone, Debug)] -pub struct ValidatedRoleGroupConfig { - pub resources: Resources, - pub logging: ValidatedLogging, - pub affinity: StackableAffinity, - pub graceful_shutdown_timeout: Duration, - pub config_file_content: String, -} - -#[derive(Clone)] -pub struct PrecomputedPodData { - pub env_vars: Vec, - pub airflow_commands: Vec, - pub auth_volumes: Vec, - pub auth_volume_mounts: Vec, - pub extra_volumes: Vec, - pub extra_volume_mounts: Vec, - pub git_sync_containers: Vec, - pub git_sync_init_containers: Vec, - pub git_sync_volumes: Vec, - pub git_sync_volume_mounts: Vec, - pub vector_container: Option, - pub service_account_name: String, - pub replicas: Option, - pub pod_overrides: PodTemplateSpec, - pub executor: AirflowExecutor, - pub executor_template_configmap_name: Option, - pub listener_volume_claim_template: Option, -} - -#[derive(Clone, Debug)] -pub struct ValidatedLogging { - pub airflow_container: ValidatedContainerLogConfigChoice, - pub vector_container: Option, - pub git_sync_container_log_config: ContainerLogConfig, -} - -impl ValidatedLogging { - pub fn is_vector_agent_enabled(&self) -> bool { - self.vector_container.is_some() - } -} - -// REVIEW: ValidatedAirflowCluster is the central validated type. It holds all data needed -// by the build stage so that build() can be infallible. All optional-after-merge fields -// are unwrapped during validation, and logging is pre-validated into ValidatedLogging. -#[derive(Clone)] -pub struct ValidatedAirflowCluster { - metadata: ObjectMeta, - pub image: ResolvedProductImage, - pub name: ClusterName, - pub namespace: NamespaceName, - pub uid: Uid, - pub role_groups: BTreeMap>, - pub precomputed_pod_data: BTreeMap>, - pub executor_template_config_maps: Vec, - pub role_configs: BTreeMap, - pub executor: AirflowExecutor, -} - -impl ValidatedAirflowCluster { - #[allow(clippy::too_many_arguments)] - pub fn new( - image: ResolvedProductImage, - name: ClusterName, - namespace: NamespaceName, - uid: Uid, - role_groups: BTreeMap>, - precomputed_pod_data: BTreeMap>, - executor_template_config_maps: Vec, - role_configs: BTreeMap, - executor: AirflowExecutor, - ) -> Self { - Self { - metadata: ObjectMeta { - name: Some(name.to_string()), - namespace: Some(namespace.to_string()), - uid: Some(uid.to_string()), - ..ObjectMeta::default() - }, - image, - name, - namespace, - uid, - role_groups, - precomputed_pod_data, - executor_template_config_maps, - role_configs, - executor, - } - } - - pub fn rolegroup_ref(&self, role: &AirflowRole, role_group: &str) -> RoleGroupRef { - RoleGroupRef { - cluster: ObjectRef::from_obj(self), - role: role.to_string(), - role_group: role_group.to_string(), - } - } -} - -impl HasName for ValidatedAirflowCluster { - fn to_name(&self) -> String { - self.name.to_string() - } -} - -impl HasUid for ValidatedAirflowCluster { - fn to_uid(&self) -> Uid { - self.uid.clone() - } -} - -impl Resource for ValidatedAirflowCluster { - type DynamicType = - ::DynamicType; - type Scope = ::Scope; - - fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha2::AirflowCluster::kind(dt) - } - - fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha2::AirflowCluster::group(dt) - } - - fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha2::AirflowCluster::version(dt) - } - - fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha2::AirflowCluster::plural(dt) - } - - fn meta(&self) -> &ObjectMeta { - &self.metadata - } - - fn meta_mut(&mut self) -> &mut ObjectMeta { - &mut self.metadata - } -} - -impl NameIsValidLabelValue for ValidatedAirflowCluster { - fn to_label_value(&self) -> String { - self.name.to_label_value() - } -} - -// --------------------------------------------------------------------------- -// Error types and reconcile -// --------------------------------------------------------------------------- - -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -pub enum Error { - #[snafu(display("AirflowCluster object is invalid"))] - InvalidAirflowCluster { - source: error_boundary::InvalidObject, - }, - - #[snafu(display("failed to dereference resources"))] - Dereference { source: DereferenceError }, - - #[snafu(display("failed to validate cluster"))] - Validate { source: ValidateError }, - - #[snafu(display("failed to create cluster resources"))] - CreateClusterResources { - source: stackable_operator::cluster_resources::Error, - }, - - #[snafu(display("failed to apply resources"))] - Apply { source: ApplyError }, - - #[snafu(display("failed to update status"))] - UpdateStatus { source: UpdateStatusError }, -} - -type Result = std::result::Result; - -impl ReconcilerError for Error { - fn category(&self) -> &'static str { - ErrorDiscriminants::from(self).into() - } -} - -// REVIEW: The reconcile pipeline is structured as five sequential stages: -// 1. dereference — async, fallible: resolve external references -// 2. validate — sync, fallible: validate and merge configs -// 3. build — sync, infallible: construct Kubernetes resources -// 4. apply — async, fallible: apply resources to the cluster -// 5. update_status — async, fallible: patch status on the CRD -pub async fn reconcile( - airflow: Arc>, - ctx: Arc, -) -> Result { - tracing::info!("Starting reconcile"); - - let airflow = airflow - .0 - .as_ref() - .map_err(error_boundary::InvalidObject::clone) - .context(InvalidAirflowClusterSnafu)?; - - // --- dereference (async, fallible) --- - let dereferenced = dereference( - &ctx.client, - airflow, - CONTAINER_IMAGE_BASE_NAME, - &ctx.operator_environment.image_repository, - crate::built_info::PKG_VERSION, - ) - .await - .context(DereferenceSnafu)?; - - // --- validate (sync, fallible) --- - let validated = - validate_cluster(airflow, &dereferenced, &ctx.product_config).context(ValidateSnafu)?; - - // REVIEW: build() is infallible — all validation and fallible operations (config - // generation, PodBuilder/ContainerBuilder usage, logging validation) happen in the - // validate stage. The build stage purely assembles Kubernetes resource structs. - // --- build (sync, infallible) --- - let prepared = build(&validated); - - // --- apply (async, fallible) --- - let cluster_resources = ClusterResources::new( - APP_NAME, - OPERATOR_NAME, - AIRFLOW_CONTROLLER_NAME, - &airflow.object_ref(&()), - ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), - &airflow.spec.object_overrides, - ) - .context(CreateClusterResourcesSnafu)?; - - let applied = Applier::new(&ctx.client, cluster_resources) - .apply(prepared) - .await - .context(ApplySnafu)?; - - // --- update status (async, fallible) --- - update_status(&ctx.client, airflow, applied) - .await - .context(UpdateStatusSnafu)?; - - Ok(Action::await_change()) -} - -pub fn error_policy( - _obj: Arc>, - error: &Error, - _ctx: Arc, -) -> Action { - match error { - Error::InvalidAirflowCluster { .. } => Action::await_change(), - _ => Action::requeue(*Duration::from_secs(10)), - } -} - -// --------------------------------------------------------------------------- -// Dereference -// --------------------------------------------------------------------------- - -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -pub enum DereferenceError { - #[snafu(display("failed to resolve product image"))] - ResolveProductImage { - source: stackable_operator::commons::product_image_selection::Error, - }, - - #[snafu(display("failed to apply authentication configuration"))] - InvalidAuthenticationConfig { - source: crate::crd::authentication::Error, - }, - - #[snafu(display("invalid authorization config"))] - InvalidAuthorizationConfig { - source: stackable_operator::commons::opa::Error, - }, - - #[snafu(display("failed to create internal secret"))] - InvalidInternalSecret { - source: random_secret_creation::Error, - }, -} - -pub struct DereferencedObjects { - pub resolved_product_image: ResolvedProductImage, - pub authentication_config: AirflowClientAuthenticationDetailsResolved, - pub authorization_config: AirflowAuthorizationResolved, -} - -pub async fn dereference( - client: &stackable_operator::client::Client, - airflow: &v1alpha2::AirflowCluster, - image_base_name: &str, - image_repository: &str, - pkg_version: &str, -) -> std::result::Result { - let resolved_product_image = airflow - .spec - .image - .resolve(image_base_name, image_repository, pkg_version) - .context(deref_err::ResolveProductImageSnafu)?; - - let authentication_config = AirflowClientAuthenticationDetailsResolved::from( - &airflow.spec.cluster_config.authentication, - client, - ) - .await - .context(deref_err::InvalidAuthenticationConfigSnafu)?; - - let authorization_config = AirflowAuthorizationResolved::from_authorization_config( - client, - airflow, - &airflow.spec.cluster_config.authorization, - ) - .await - .context(deref_err::InvalidAuthorizationConfigSnafu)?; - - random_secret_creation::create_random_secret_if_not_exists( - &airflow.shared_internal_secret_secret_name(), - INTERNAL_SECRET_SECRET_KEY, - 256, - airflow, - client, - ) - .await - .context(deref_err::InvalidInternalSecretSnafu)?; - - random_secret_creation::create_random_secret_if_not_exists( - &airflow.shared_jwt_secret_secret_name(), - JWT_SECRET_SECRET_KEY, - 256, - airflow, - client, - ) - .await - .context(deref_err::InvalidInternalSecretSnafu)?; - - random_secret_creation::create_random_secret_if_not_exists( - &airflow.shared_fernet_key_secret_name(), - FERNET_KEY_SECRET_KEY, - 32, - airflow, - client, - ) - .await - .context(deref_err::InvalidInternalSecretSnafu)?; - - Ok(DereferencedObjects { - resolved_product_image, - authentication_config, - authorization_config, - }) -} - -/// Module-like namespace for Snafu context selectors for DereferenceError, -/// avoiding name collisions with the top-level and validate error selectors. -mod deref_err { - pub(super) use super::{ - InvalidAuthenticationConfigSnafu, InvalidAuthorizationConfigSnafu, - InvalidInternalSecretSnafu, ResolveProductImageSnafu, - }; -} - -// --------------------------------------------------------------------------- -// Validate -// --------------------------------------------------------------------------- - -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -pub enum ValidateError { - #[snafu(display("failed to validate cluster name"))] - InvalidClusterName { - source: crate::framework::macros::attributed_string_type::Error, - }, - - #[snafu(display("object has no associated namespace"))] - ObjectHasNoNamespace, - - #[snafu(display("failed to validate cluster namespace"))] - InvalidClusterNamespace { - source: crate::framework::macros::attributed_string_type::Error, - }, - - #[snafu(display("object has no UID"))] - ObjectHasNoUid, - - #[snafu(display("failed to validate cluster UID"))] - InvalidClusterUid { - source: crate::framework::macros::attributed_string_type::Error, - }, - - #[snafu(display("failed to validate logging configuration"))] - ValidateLoggingConfig { - source: crate::framework::product_logging::framework::Error, - }, - - #[snafu(display("vectorAggregatorConfigMapName must be set when vector agent is enabled"))] - MissingVectorAggregatorConfigMapName, - - #[snafu(display("failed to parse vector aggregator ConfigMap name"))] - ParseVectorAggregatorConfigMapName { - source: crate::framework::macros::attributed_string_type::Error, - }, - - #[snafu(display("graceful shutdown timeout is not configured"))] - MissingGracefulShutdownTimeout, - - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("failed to construct Airflow configuration"))] - ConstructConfig { source: crate::config::Error }, - - #[snafu(display("failed to write config file"))] - BuildConfigFile { - source: product_config::flask_app_config_writer::FlaskAppConfigWriterError, - }, - - #[snafu(display("Failed to transform configs"))] - ProductConfigTransform { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("could not parse Airflow role [{role}]"))] - UnidentifiedAirflowRole { - source: strum::ParseError, - role: String, - }, - - #[snafu(display("object defines no airflow config role"))] - NoAirflowRole, - - #[snafu(display("failed to build environment variables"))] - BuildEnvVars { source: crate::env_vars::Error }, - - #[snafu(display("invalid git-sync specification"))] - InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, - - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - - #[snafu(display("failed to add LDAP volumes and volume mounts"))] - AddLdapVolumesAndVolumeMounts { - source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, - }, - - #[snafu(display("failed to add TLS volumes and volume mounts"))] - AddTlsVolumesAndVolumeMounts { - source: stackable_operator::commons::tls_verification::TlsClientDetailsError, - }, - - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, - }, - - #[snafu(display("failed to build labels"))] - BuildLabels { - source: stackable_operator::kvp::LabelError, - }, - - #[snafu(display("invalid container name"))] - InvalidContainerName { - source: stackable_operator::builder::pod::container::Error, - }, - - #[snafu(display("failed to add volume mount"))] - AddVolumeMount { - source: stackable_operator::builder::pod::container::Error, - }, - - #[snafu(display("failed to add volume"))] - AddVolume { - source: stackable_operator::builder::pod::Error, - }, - - #[snafu(display("failed to serialize pod template"))] - PodTemplateSerde { source: serde_yaml::Error }, - - #[snafu(display("failed to build pod template ConfigMap"))] - PodTemplateConfigMap { - source: stackable_operator::builder::configmap::Error, - }, - - #[snafu(display("failed to build object metadata"))] - ObjectMeta { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build graceful shutdown config"))] - GracefulShutdown { - source: stackable_operator::builder::pod::Error, - }, -} - -type ValidateResult = std::result::Result; - -fn validate_logging( - logging: &Logging, - main_container: Container, - vector_aggregator_config_map_name: Option<&str>, -) -> ValidateResult { - let airflow_container = validate_logging_configuration_for_container(logging, main_container) - .context(validate_err::ValidateLoggingConfigSnafu)?; - - let vector_container = if logging.enable_vector_agent { - let aggregator_name = vector_aggregator_config_map_name - .context(validate_err::MissingVectorAggregatorConfigMapNameSnafu)?; - ConfigMapName::from_str(aggregator_name) - .context(validate_err::ParseVectorAggregatorConfigMapNameSnafu)?; - let log_config = validate_logging_configuration_for_container(logging, Container::Vector) - .context(validate_err::ValidateLoggingConfigSnafu)?; - Some(VectorContainerLogConfig { log_config }) - } else { - None - }; - - let git_sync_container_log_config = logging.for_container(&Container::GitSync).into_owned(); - - Ok(ValidatedLogging { - airflow_container, - vector_container, - git_sync_container_log_config, - }) -} - -fn validate_airflow_config( - config: &AirflowConfig, - vector_aggregator_config_map_name: Option<&str>, - config_file_content: String, -) -> ValidateResult { - let logging = validate_logging( - &config.logging, - Container::Airflow, - vector_aggregator_config_map_name, - )?; - - let graceful_shutdown_timeout = config - .graceful_shutdown_timeout - .context(validate_err::MissingGracefulShutdownTimeoutSnafu)?; - - Ok(ValidatedRoleGroupConfig { - resources: config.resources.clone(), - logging, - affinity: config.affinity.clone(), - graceful_shutdown_timeout, - config_file_content, - }) -} - -fn validate_executor_config( - config: &ExecutorConfig, - vector_aggregator_config_map_name: Option<&str>, - config_file_content: String, -) -> ValidateResult { - let logging = validate_logging( - &config.logging, - Container::Base, - vector_aggregator_config_map_name, - )?; - - let graceful_shutdown_timeout = config - .graceful_shutdown_timeout - .context(validate_err::MissingGracefulShutdownTimeoutSnafu)?; - - Ok(ValidatedRoleGroupConfig { - resources: config.resources.clone(), - logging, - affinity: config.affinity.clone(), - graceful_shutdown_timeout, - config_file_content, - }) -} - -/// Generates the `webserver_config.py` content for a role group. -/// -/// This function is called during the validate stage so that the build stage can -/// construct ConfigMaps infallibly. -fn generate_config_file_content( - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, - product_version: &str, - rolegroup_config_overrides: &HashMap< - PropertyNameKind, - BTreeMap, - >, -) -> ValidateResult { - use std::io::Write; - - use product_config::flask_app_config_writer; - use stackable_operator::product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, - }; - - use crate::{ - config::{self, PYTHON_IMPORTS}, - crd::{AIRFLOW_CONFIG_FILENAME, AirflowConfigOptions}, - }; - - let mut config = BTreeMap::new(); - config::add_airflow_config( - &mut config, - authentication_config, - authorization_config, - product_version, - ) - .context(validate_err::ConstructConfigSnafu)?; - - let mut file_overrides = rolegroup_config_overrides - .get(&PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.to_string())) - .cloned() - .unwrap_or_default(); - - config.append(&mut file_overrides); - - let mut config_file = Vec::new(); - - if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).expect("writing to Vec is infallible"); - } - - let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - - flask_app_config_writer::write::( - &mut config_file, - config.iter(), - PYTHON_IMPORTS, - ) - .context(validate_err::BuildConfigFileSnafu)?; - - if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).expect("writing to Vec is infallible"); - } - - Ok(String::from_utf8(config_file).expect("flask_app_config_writer produces valid UTF-8")) -} - -/// Top-level validation: runs product_config, merges/validates per-rolegroup configs, -/// generates config file contents, and assembles a [`ValidatedAirflowCluster`]. -fn validate_cluster( - airflow: &v1alpha2::AirflowCluster, - dereferenced: &DereferencedObjects, - product_config_manager: &ProductConfigManager, -) -> ValidateResult { - let vector_aggregator_config_map_name = airflow - .spec - .cluster_config - .vector_aggregator_config_map_name - .as_deref(); - - // --- product_config transform + validate --- - let mut roles = HashMap::new(); - for role in AirflowRole::iter() { - if let Some(resolved_role) = airflow.get_role(&role) { - roles.insert( - role.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.into()), - ], - resolved_role.clone(), - ), - ); - } - } - - let role_config = transform_all_roles_to_config(airflow, &roles); - let validated_role_config = validate_all_roles_and_groups_config( - &dereferenced.resolved_product_image.product_version, - &role_config.context(validate_err::ProductConfigTransformSnafu)?, - product_config_manager, - false, - false, - ) - .context(validate_err::InvalidProductConfigSnafu)?; - - // --- compute database connection details (infallible) --- - let templating_mechanism = - stackable_operator::database_connections::TemplatingMechanism::BashEnvSubstitution; - let metadata_database_connection_details = airflow - .spec - .cluster_config - .metadata_database - .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); - let celery_database_connection_details = if let ( - Some(celery_results_backend), - Some(celery_broker), - ) = ( - &airflow.spec.cluster_config.celery_results_backend, - &airflow.spec.cluster_config.celery_broker, - ) { - // The celery results backend and celery broker only work with configured celeryExecutors. - // Emit a warning if celery executors were not configured properly. - if !matches!( - &airflow.spec.executor, - AirflowExecutor::CeleryExecutors { .. } - ) { - tracing::warn!( - "No `spec.celeryExecutors` configured, but `spec.clusterConfig.celeryResultsBackend` and `spec.clusterConfig.celeryBroker` are provided. This only works in combination with a celery executor!" - ) - } - let celery_results_backend = celery_results_backend - .celery_connection_details_with_templating( - "CELERY_RESULT_BACKEND", - &templating_mechanism, - ); - let celery_broker = celery_broker - .celery_connection_details_with_templating("CELERY_BROKER", &templating_mechanism); - Some((celery_results_backend, celery_broker)) - } else { - None - }; - - // --- compute auth volumes/mounts (fallible) --- - let (auth_volumes, auth_volume_mounts) = - compute_auth_volumes_and_mounts(&dereferenced.authentication_config)?; - - // --- service account name (matches build_rbac_resources output: "{cluster}-serviceaccount") --- - let service_account_name = format!("{}-serviceaccount", airflow.name_any()); - - // --- per-role/rolegroup validation --- - let mut validated_role_groups = BTreeMap::new(); - let mut all_precomputed_pod_data = BTreeMap::new(); - - for (role_name, role_config) in validated_role_config.iter() { - let airflow_role = - AirflowRole::from_str(role_name).context(validate_err::UnidentifiedAirflowRoleSnafu { - role: role_name.to_string(), - })?; - - let mut validated_groups = BTreeMap::new(); - let mut pod_data_groups = BTreeMap::new(); - - for (rolegroup_name, rolegroup_config) in role_config.iter() { - let rolegroup_ref = RoleGroupRef { - cluster: ObjectRef::from_obj(airflow), - role: role_name.into(), - role_group: rolegroup_name.into(), - }; - - let merged_airflow_config = airflow - .merged_config(&airflow_role, &rolegroup_ref) - .context(validate_err::FailedToResolveConfigSnafu)?; - - let config_file_content = generate_config_file_content( - &dereferenced.authentication_config, - &dereferenced.authorization_config, - &dereferenced.resolved_product_image.product_version, - rolegroup_config, - )?; - - let validated_config = validate_airflow_config( - &merged_airflow_config, - vector_aggregator_config_map_name, - config_file_content, - )?; - - let pod_data = compute_precomputed_pod_data( - airflow, - &airflow_role, - &rolegroup_ref, - rolegroup_config, - &dereferenced.resolved_product_image, - &dereferenced.authentication_config, - &dereferenced.authorization_config, - &metadata_database_connection_details, - &celery_database_connection_details, - &validated_config.logging, - &auth_volumes, - &auth_volume_mounts, - &service_account_name, - )?; - - validated_groups.insert(rolegroup_name.clone(), validated_config); - pod_data_groups.insert(rolegroup_name.clone(), pod_data); - } - - validated_role_groups.insert(airflow_role.clone(), validated_groups); - all_precomputed_pod_data.insert(airflow_role, pod_data_groups); - } - - // --- per-role config (PDB, listeners) --- - let mut validated_role_configs_map = BTreeMap::new(); - for role in AirflowRole::iter() { - if let Some(role_config) = airflow.role_config(&role) { - let pdb = &role_config.pod_disruption_budget; - let listener_class = role.listener_class_name(airflow); - let group_listener_name = airflow.group_listener_name(&role); - validated_role_configs_map.insert( - role, - ValidatedRoleConfig { - pdb_enabled: pdb.enabled, - pdb_max_unavailable: pdb.max_unavailable, - listener_class, - group_listener_name, - }, - ); - } - } - - // --- executor template config maps --- - let executor_template_config_maps = if let AirflowExecutor::KubernetesExecutors { - common_configuration, - } = &airflow.spec.executor - { - let merged_executor_config = airflow - .merged_executor_config(&common_configuration.config) - .context(validate_err::FailedToResolveConfigSnafu)?; - - let config_file_content = generate_config_file_content( - &dereferenced.authentication_config, - &dereferenced.authorization_config, - &dereferenced.resolved_product_image.product_version, - &HashMap::new(), - )?; - - let validated_config = validate_executor_config( - &merged_executor_config, - vector_aggregator_config_map_name, - config_file_content, - )?; - - build_executor_template_config_maps( - airflow, - &dereferenced.resolved_product_image, - &dereferenced.authentication_config, - &metadata_database_connection_details, - &service_account_name, - &validated_config, - common_configuration, - )? - } else { - Vec::new() - }; - - // --- assemble --- - validate_and_assemble( - airflow, - &dereferenced.resolved_product_image, - validated_role_groups, - all_precomputed_pod_data, - executor_template_config_maps, - validated_role_configs_map, - ) -} - -/// Validates the AirflowCluster and produces a [`ValidatedAirflowCluster`] containing -/// all role groups with their validated configs. -fn validate_and_assemble( - airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - validated_role_configs: BTreeMap>, - precomputed_pod_data: BTreeMap>, - executor_template_config_maps: Vec, - role_configs: BTreeMap, -) -> ValidateResult { - let cluster_name = - ClusterName::from_str(&airflow.name_any()).context(validate_err::InvalidClusterNameSnafu)?; - let namespace = NamespaceName::from_str( - &airflow - .namespace() - .context(validate_err::ObjectHasNoNamespaceSnafu)?, - ) - .context(validate_err::InvalidClusterNamespaceSnafu)?; - let uid = Uid::from_str( - airflow - .meta() - .uid - .as_deref() - .context(validate_err::ObjectHasNoUidSnafu)?, - ) - .context(validate_err::InvalidClusterUidSnafu)?; - - Ok(ValidatedAirflowCluster::new( - resolved_product_image.clone(), - cluster_name, - namespace, - uid, - validated_role_configs, - precomputed_pod_data, - executor_template_config_maps, - role_configs, - airflow.spec.executor.clone(), - )) -} - -/// Extracts auth volumes and volume mounts using temporary builders. -/// -/// The upstream LDAP/TLS provider APIs require `PodBuilder`/`ContainerBuilder` references. -/// We create temporary builders, call the auth methods, then extract the raw volumes and mounts. -fn compute_auth_volumes_and_mounts( - authentication_config: &AirflowClientAuthenticationDetailsResolved, -) -> ValidateResult<(Vec, Vec)> { - let mut pb = PodBuilder::new(); - let mut cb = ContainerBuilder::new("dummy").expect("'dummy' is a valid container name"); - - let mut ldap_providers = BTreeSet::new(); - let mut tls_credentials = BTreeSet::new(); - - for auth_class in &authentication_config.authentication_classes_resolved { - match auth_class { - AirflowAuthenticationClassResolved::Ldap { provider } => { - ldap_providers.insert(provider); - } - AirflowAuthenticationClassResolved::Oidc { provider, .. } => { - tls_credentials.insert(&provider.tls); - } - } - } - - for provider in ldap_providers { - provider - .add_volumes_and_mounts(&mut pb, vec![&mut cb]) - .context(validate_err::AddLdapVolumesAndVolumeMountsSnafu)?; - } - for tls in tls_credentials { - tls.add_volumes_and_mounts(&mut pb, vec![&mut cb]) - .context(validate_err::AddTlsVolumesAndVolumeMountsSnafu)?; - } - - let container = cb.build(); - let pod_template = pb.build_template(); - - let volumes = pod_template - .spec - .and_then(|s| s.volumes) - .unwrap_or_default(); - let mounts = container.volume_mounts.unwrap_or_default(); - - Ok((volumes, mounts)) -} - -/// Builds the executor template ConfigMaps for KubernetesExecutor mode. -/// -/// Produces two ConfigMaps: -/// 1. A logging/config ConfigMap for the executor pods (equivalent to a rolegroup ConfigMap) -/// 2. A pod template ConfigMap containing a serialised PodTemplate that Airflow uses to -/// launch executor pods -/// -/// This is done in the validate stage because it uses PodBuilder/ContainerBuilder which -/// are fallible. The build stage then just passes these through to KubernetesResources. -#[allow(clippy::too_many_arguments)] -fn build_executor_template_config_maps( - airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - service_account_name: &str, - validated_config: &ValidatedRoleGroupConfig, - common_configuration: &crate::crd::AirflowExecutorCommonConfiguration, -) -> ValidateResult> { - let executor_rolegroup_ref = RoleGroupRef { - cluster: ObjectRef::from_obj(airflow), - role: "executor".into(), - role_group: "kubernetes".into(), - }; - - // 1. Build the executor logging/config ConfigMap - let executor_config_cm = { - let metadata = ObjectMetaBuilder::new() - .name(executor_rolegroup_ref.object_name()) - .namespace_opt(airflow.namespace()) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(validate_err::ObjectMetaSnafu)? - .with_recommended_labels(&build_object_labels( - airflow, - resolved_product_image, - "executor", - "executor-template", - )) - .context(validate_err::ObjectMetaSnafu)? - .build(); - - let mut cm_builder = ConfigMapBuilder::new(); - cm_builder.metadata(metadata); - cm_builder.add_data( - AIRFLOW_CONFIG_FILENAME, - validated_config.config_file_content.clone(), - ); - - extend_config_map_with_log_config( - &executor_rolegroup_ref, - &Container::Base, - &validated_config.logging.airflow_container, - validated_config.logging.vector_container.as_ref(), - &mut cm_builder, - resolved_product_image, - ); - - cm_builder - .build() - .context(validate_err::PodTemplateConfigMapSnafu)? - }; - - // 2. Build the executor pod template ConfigMap - let executor_template_cm = { - // git-sync resources for the executor template - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( - &airflow.spec.cluster_config.dags_git_sync, - resolved_product_image, - &env_vars_from(&common_configuration.env_overrides), - &airflow.volume_mounts(), - LOG_VOLUME_NAME, - &validated_config.logging.git_sync_container_log_config, - ) - .context(validate_err::InvalidGitSyncSpecSnafu)?; - - let mut pb = PodBuilder::new(); - let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&build_object_labels( - airflow, - resolved_product_image, - "executor", - "executor-template", - )) - .context(validate_err::ObjectMetaSnafu)? - .build(); - - pb.metadata(pb_metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .affinity(&validated_config.affinity) - .service_account_name(service_account_name) - .restart_policy("Never") - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - - pb.termination_grace_period(&validated_config.graceful_shutdown_timeout) - .context(validate_err::GracefulShutdownSnafu)?; - - // Container name "base" is an Airflow requirement - let mut airflow_container = ContainerBuilder::new(&Container::Base.to_string()) - .context(validate_err::InvalidContainerNameSnafu)?; - - // Auth volumes and mounts - add_authentication_volumes_and_volume_mounts_to_builders( - authentication_config, - &mut airflow_container, - &mut pb, - )?; - - airflow_container - .image_from_product_image(resolved_product_image) - .resources(validated_config.resources.clone().into()) - .add_env_vars(env_vars::build_airflow_template_envs( - airflow, - &common_configuration.env_overrides, - validated_config.logging.is_vector_agent_enabled(), - metadata_database_connection_details, - &git_sync_resources, - resolved_product_image, - )) - .add_volume_mounts(airflow.volume_mounts()) - .context(validate_err::AddVolumeMountSnafu)? - .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) - .context(validate_err::AddVolumeMountSnafu)? - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR) - .context(validate_err::AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(validate_err::AddVolumeMountSnafu)?; - - // Git-sync resources (init containers only, no sidecars for executor template) - for container in git_sync_resources.git_sync_init_containers.iter().cloned() { - pb.add_init_container(container); - } - pb.add_volumes(git_sync_resources.git_content_volumes.clone()) - .context(validate_err::AddVolumeSnafu)?; - pb.add_volumes(git_sync_resources.git_ssh_volumes.clone()) - .context(validate_err::AddVolumeSnafu)?; - pb.add_volumes(git_sync_resources.git_ca_cert_volumes.clone()) - .context(validate_err::AddVolumeSnafu)?; - airflow_container - .add_volume_mounts(git_sync_resources.git_content_volume_mounts.clone()) - .context(validate_err::AddVolumeMountSnafu)?; - - // Database connection env vars - metadata_database_connection_details.add_to_container(&mut airflow_container); - - pb.add_container(airflow_container.build()); - pb.add_volumes(airflow.volumes().clone()) - .context(validate_err::AddVolumeSnafu)?; - // REVIEW: controller_commons::create_volumes now takes &ValidatedContainerLogConfigChoice - // instead of Option<&ContainerLogConfig>. The validated type ensures logging config - // has already been checked, so the build stage can use it directly. - pb.add_volumes(controller_commons::create_volumes( - &executor_rolegroup_ref.object_name(), - &validated_config.logging.airflow_container, - )) - .context(validate_err::AddVolumeSnafu)?; - - if let Some(vector_config) = &validated_config.logging.vector_container { - let vector_aggregator_config_map_name = airflow - .spec - .cluster_config - .vector_aggregator_config_map_name - .as_deref() - .context(validate_err::MissingVectorAggregatorConfigMapNameSnafu)?; - pb.add_container(build_logging_container( - resolved_product_image, - vector_config, - vector_aggregator_config_map_name, - )?); - } - - let mut pod_template = pb.build_template(); - pod_template.merge_from(common_configuration.pod_overrides.clone()); - - let restarter_label = Label::try_from(("restarter.stackable.tech/enabled", "true")) - .expect("static label is always valid"); - - let mut cm_builder = ConfigMapBuilder::new(); - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(airflow.executor_template_configmap_name()) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(validate_err::ObjectMetaSnafu)? - .with_recommended_labels(&build_object_labels( - airflow, - resolved_product_image, - "executor", - "executor-template", - )) - .context(validate_err::ObjectMetaSnafu)? - .with_label(restarter_label) - .build(), - ) - .add_data( - TEMPLATE_NAME, - serde_yaml::to_string(&pod_template) - .context(validate_err::PodTemplateSerdeSnafu)?, - ); - - cm_builder - .build() - .context(validate_err::PodTemplateConfigMapSnafu)? - }; - - Ok(vec![executor_config_cm, executor_template_cm]) -} - -/// Helper to add authentication volumes and volume mounts directly to builders. -/// Used by the executor template where we build a PodTemplate using PodBuilder/ContainerBuilder. -fn add_authentication_volumes_and_volume_mounts_to_builders( - authentication_config: &AirflowClientAuthenticationDetailsResolved, - cb: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> ValidateResult<()> { - let mut ldap_providers = BTreeSet::new(); - let mut tls_credentials = BTreeSet::new(); - - for auth_class in &authentication_config.authentication_classes_resolved { - match auth_class { - AirflowAuthenticationClassResolved::Ldap { provider } => { - ldap_providers.insert(provider); - } - AirflowAuthenticationClassResolved::Oidc { provider, .. } => { - tls_credentials.insert(&provider.tls); - } - } - } - - for provider in ldap_providers { - provider - .add_volumes_and_mounts(pb, vec![cb]) - .context(validate_err::AddLdapVolumesAndVolumeMountsSnafu)?; - } - for tls in tls_credentials { - tls.add_volumes_and_mounts(pb, vec![cb]) - .context(validate_err::AddTlsVolumesAndVolumeMountsSnafu)?; - } - Ok(()) -} - -fn build_object_labels<'a>( - airflow: &'a v1alpha2::AirflowCluster, - resolved_product_image: &'a ResolvedProductImage, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, v1alpha2::AirflowCluster> { - ObjectLabels { - owner: airflow, - app_name: APP_NAME, - app_version: &resolved_product_image.app_version_label_value, - operator_name: OPERATOR_NAME, - controller_name: AIRFLOW_CONTROLLER_NAME, - role, - role_group, - } -} - -fn build_logging_container( - resolved_product_image: &ResolvedProductImage, - vector_config: &VectorContainerLogConfig, - vector_aggregator_config_map_name: &str, -) -> ValidateResult { - let raw_log_config = vector_config.log_config.to_raw_container_log_config(); - - product_logging::framework::vector_container( - resolved_product_image, - CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - Some(&raw_log_config), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(validate_err::ConfigureLoggingSnafu) -} - -/// Computes all pod-level data needed by the build stage to construct StatefulSets infallibly. -#[allow(clippy::too_many_arguments)] -fn compute_precomputed_pod_data( - airflow: &v1alpha2::AirflowCluster, - airflow_role: &AirflowRole, - rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, - resolved_product_image: &ResolvedProductImage, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, - metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - celery_database_connection_details: &Option<( - CeleryDatabaseConnectionDetails, - CeleryDatabaseConnectionDetails, - )>, - validated_logging: &ValidatedLogging, - auth_volumes: &[Volume], - auth_volume_mounts: &[VolumeMount], - service_account_name: &str, -) -> ValidateResult { - let executor = &airflow.spec.executor; - - // --- git-sync resources --- - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( - &airflow.spec.cluster_config.dags_git_sync, - resolved_product_image, - &env_vars_from_rolegroup_config(rolegroup_config), - &airflow.volume_mounts(), - LOG_VOLUME_NAME, - &validated_logging.git_sync_container_log_config, - ) - .context(validate_err::InvalidGitSyncSpecSnafu)?; - - // --- env vars --- - let mut env_vars = env_vars::build_airflow_statefulset_envs( - airflow, - airflow_role, - rolegroup_config, - executor, - authentication_config, - authorization_config, - metadata_database_connection_details, - celery_database_connection_details, - &git_sync_resources, - resolved_product_image, - ) - .context(validate_err::BuildEnvVarsSnafu)?; - - // Database connection details add secret-referenced env vars via ContainerBuilder. - // Extract them using a temp builder. - let db_env_vars = { - let mut cb = ContainerBuilder::new("dummy").expect("'dummy' is a valid container name"); - metadata_database_connection_details.add_to_container(&mut cb); - if let Some((celery_result_backend, celery_broker)) = celery_database_connection_details { - celery_result_backend.add_to_container(&mut cb); - celery_broker.add_to_container(&mut cb); - } - cb.build().env.unwrap_or_default() - }; - env_vars.extend(db_env_vars); - - // --- commands --- - let airflow_commands = - airflow_role.get_commands(airflow, authentication_config, resolved_product_image); - - // --- git-sync containers/volumes --- - let use_git_sync_init_containers = matches!(executor, AirflowExecutor::CeleryExecutors { .. }); - let git_sync_containers = git_sync_resources.git_sync_containers.clone(); - let git_sync_init_containers = if use_git_sync_init_containers { - git_sync_resources.git_sync_init_containers.clone() - } else { - Vec::new() - }; - let mut git_sync_volumes = git_sync_resources.git_content_volumes.clone(); - git_sync_volumes.extend(git_sync_resources.git_ssh_volumes.clone()); - git_sync_volumes.extend(git_sync_resources.git_ca_cert_volumes.clone()); - let git_sync_volume_mounts = git_sync_resources.git_content_volume_mounts.clone(); - - // --- vector container --- - let vector_container = if let Some(vector_config) = &validated_logging.vector_container { - let vector_aggregator_config_map_name = airflow - .spec - .cluster_config - .vector_aggregator_config_map_name - .as_deref() - .context(validate_err::MissingVectorAggregatorConfigMapNameSnafu)?; - Some(build_logging_container( - resolved_product_image, - vector_config, - vector_aggregator_config_map_name, - )?) - } else { - None - }; - - // --- replicas --- - let binding = airflow.get_role(airflow_role); - let role = binding.as_ref().context(validate_err::NoAirflowRoleSnafu)?; - let rolegroup = role.role_groups.get(&rolegroup_ref.role_group); - let replicas = rolegroup.and_then(|rg| rg.replicas); - - // --- pod overrides --- - let mut pod_overrides = PodTemplateSpec::default(); - pod_overrides.merge_from(role.config.pod_overrides.clone()); - if let Some(rg) = rolegroup { - pod_overrides.merge_from(rg.config.pod_overrides.clone()); - } - - // --- executor template configmap name --- - let executor_template_configmap_name = - if matches!(executor, AirflowExecutor::KubernetesExecutors { .. }) { - Some(airflow.executor_template_configmap_name()) - } else { - None - }; - - // --- listener PVC --- - let listener_volume_claim_template = if airflow_role.get_http_port().is_some() { - if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) { - let unversioned_labels = Labels::recommended(&ObjectLabels { - owner: airflow, - app_name: APP_NAME, - app_version: "none", - operator_name: OPERATOR_NAME, - controller_name: AIRFLOW_CONTROLLER_NAME, - role: &rolegroup_ref.role, - role_group: &rolegroup_ref.role_group, - }) - .context(validate_err::BuildLabelsSnafu)?; - - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(listener_group_name), - &unversioned_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(validate_err::BuildListenerVolumeSnafu)?; - Some(pvc) - } else { - None - } - } else { - None - }; - - // --- user-defined extra volumes/mounts from CRD --- - let extra_volumes = airflow.volumes().clone(); - let extra_volume_mounts = airflow.volume_mounts(); - - Ok(PrecomputedPodData { - env_vars, - airflow_commands, - auth_volumes: auth_volumes.to_vec(), - auth_volume_mounts: auth_volume_mounts.to_vec(), - extra_volumes, - extra_volume_mounts, - git_sync_containers, - git_sync_init_containers, - git_sync_volumes, - git_sync_volume_mounts, - vector_container, - service_account_name: service_account_name.to_string(), - replicas, - pod_overrides, - executor: executor.clone(), - executor_template_configmap_name, - listener_volume_claim_template, - }) -} - -/// Module-like namespace for Snafu context selectors for ValidateError, -/// avoiding name collisions with the top-level error selectors. -mod validate_err { - pub(super) use super::{ - AddLdapVolumesAndVolumeMountsSnafu, AddTlsVolumesAndVolumeMountsSnafu, AddVolumeMountSnafu, - AddVolumeSnafu, BuildConfigFileSnafu, BuildEnvVarsSnafu, BuildLabelsSnafu, - BuildListenerVolumeSnafu, ConfigureLoggingSnafu, ConstructConfigSnafu, - FailedToResolveConfigSnafu, GracefulShutdownSnafu, InvalidClusterNameSnafu, - InvalidClusterNamespaceSnafu, InvalidClusterUidSnafu, InvalidContainerNameSnafu, - InvalidGitSyncSpecSnafu, InvalidProductConfigSnafu, - MissingGracefulShutdownTimeoutSnafu, MissingVectorAggregatorConfigMapNameSnafu, - NoAirflowRoleSnafu, ObjectHasNoNamespaceSnafu, ObjectHasNoUidSnafu, ObjectMetaSnafu, - ParseVectorAggregatorConfigMapNameSnafu, PodTemplateConfigMapSnafu, - PodTemplateSerdeSnafu, ProductConfigTransformSnafu, UnidentifiedAirflowRoleSnafu, - ValidateLoggingConfigSnafu, - }; -} - -// --------------------------------------------------------------------------- -// Build (including RoleGroupBuilder) -// --------------------------------------------------------------------------- - -fn main_container_for_role(_role: &AirflowRole) -> Container { - Container::Airflow -} - -// REVIEW: build() is infallible. All validation and fallible operations (config generation, -// PodBuilder/ContainerBuilder usage, logging validation) are performed in the validate -// stage. The build stage purely assembles Kubernetes resource structs from validated data. -fn build(validated: &ValidatedAirflowCluster) -> KubernetesResources { - let mut stateful_sets = Vec::new(); - let mut config_maps = Vec::new(); - let mut services = Vec::new(); - let mut pod_disruption_budgets = Vec::new(); - let mut listeners = Vec::new(); - - // --- RBAC --- - let rbac_labels = build_recommended_labels(validated, "rbac", "rbac"); - - let (rbac_sa, rbac_rolebinding) = build_rbac_resources(validated, APP_NAME, rbac_labels) - .expect( - "RBAC resources should be created because the validated cluster has valid metadata", - ); - - // --- Executor template ConfigMaps (pre-built in validate stage) --- - config_maps.extend(validated.executor_template_config_maps.clone()); - - // --- Per-role/rolegroup resources --- - for (airflow_role, role_groups) in &validated.role_groups { - // PDBs - if let Some(role_config) = validated.role_configs.get(airflow_role) { - if let Some(pdb) = build_pdb(validated, airflow_role, role_config) { - pod_disruption_budgets.push(pdb); - } - } - - // Group listeners (only Webserver) - if let Some(role_config) = validated.role_configs.get(airflow_role) { - if let (Some(listener_class), Some(listener_name)) = ( - &role_config.listener_class, - &role_config.group_listener_name, - ) { - listeners.push(build_group_listener( - validated, - airflow_role, - listener_class.clone(), - listener_name.clone(), - )); - } - } - - for (rolegroup_name, role_group_config) in role_groups { - let rolegroup_ref = validated.rolegroup_ref(airflow_role, rolegroup_name); - - let main_container = main_container_for_role(airflow_role); - - // Services - services.push(build_headless_service(validated, &rolegroup_ref)); - services.push(build_metrics_service(validated, &rolegroup_ref)); - - // ConfigMap + StatefulSet via RoleGroupBuilder - let pod_data = validated - .precomputed_pod_data - .get(airflow_role) - .and_then(|groups| groups.get(rolegroup_name)) - .expect( - "PrecomputedPodData should exist for every role group \ - because validate_cluster computes it for each one", - ); - - let builder = RoleGroupBuilder::new( - validated, - role_group_config, - rolegroup_ref, - airflow_role.clone(), - main_container, - pod_data, - ); - - config_maps.push(builder.build_config_map()); - stateful_sets.push(builder.build_stateful_set()); - } - } - - KubernetesResources { - stateful_sets, - config_maps, - services, - service_accounts: vec![rbac_sa], - role_bindings: vec![rbac_rolebinding], - pod_disruption_budgets, - listeners, - _status: PhantomData, - } -} - -fn build_pdb( - cluster: &ValidatedAirflowCluster, - role: &AirflowRole, - role_config: &ValidatedRoleConfig, -) -> Option { - if !role_config.pdb_enabled { - return None; - } - - let max_unavailable = role_config.pdb_max_unavailable.unwrap_or(match role { - AirflowRole::Worker => match &cluster.executor { - AirflowExecutor::KubernetesExecutors { .. } => return None, - _ => 1, - }, - _ => 1, - }); - - // REVIEW: from_str_unsafe is used here because the values come from constants (APP_NAME, - // OPERATOR_NAME, AIRFLOW_CONTROLLER_NAME) or validated role names — they are known to be - // valid at compile time or have been validated during the validate stage. - Some({ - use crate::framework::types::operator::*; - framework::builder::pdb::pod_disruption_budget_builder_with_role( - cluster, - &ProductName::from_str_unsafe(APP_NAME), - &RoleName::from_str_unsafe(&role.to_string()), - &OperatorName::from_str_unsafe(OPERATOR_NAME), - &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), - ) - .with_max_unavailable(max_unavailable) - .build() - }) -} - -fn build_headless_service( - cluster: &ValidatedAirflowCluster, - rolegroup_ref: &RoleGroupRef, -) -> Service { - let metadata = ObjectMetaBuilder::new() - .name(format!("{}-headless", rolegroup_ref.object_name())) - .namespace(&cluster.namespace) - .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_labels(build_recommended_labels( - cluster, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .build(); - - Service { - metadata, - spec: Some(ServiceSpec { - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(vec![ServicePort { - name: Some(HTTP_PORT_NAME.to_string()), - port: HTTP_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]), - selector: Some( - build_role_group_selector_labels( - cluster, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - } -} - -fn build_metrics_service( - cluster: &ValidatedAirflowCluster, - rolegroup_ref: &RoleGroupRef, -) -> Service { - let metadata = ObjectMetaBuilder::new() - .name(format!("{}-metrics", rolegroup_ref.object_name())) - .namespace(&cluster.namespace) - .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_labels(build_recommended_labels( - cluster, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .with_labels(prometheus_labels()) - .with_annotations(prometheus_annotations()) - .build(); - - Service { - metadata, - spec: Some(ServiceSpec { - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(vec![ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]), - selector: Some( - build_role_group_selector_labels( - cluster, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - } -} - -// REVIEW: from_str_unsafe is used for label construction throughout these helpers because -// the inputs are either compile-time constants (APP_NAME, OPERATOR_NAME, etc.) or values -// that have already been validated during the validate stage (role names, role group names, -// product versions). Using from_str_unsafe avoids redundant re-validation in the infallible -// build stage. -fn build_recommended_labels( - cluster: &ValidatedAirflowCluster, - role: &str, - role_group: &str, -) -> Labels { - use crate::framework::types::operator::*; - framework::kvp::label::recommended_labels( - cluster, - &ProductName::from_str_unsafe(APP_NAME), - &ProductVersion::from_str_unsafe(&cluster.image.app_version_label_value.to_string()), - &OperatorName::from_str_unsafe(OPERATOR_NAME), - &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), - &RoleName::from_str_unsafe(role), - &RoleGroupName::from_str_unsafe(role_group), - ) -} - -fn build_role_group_selector_labels( - cluster: &ValidatedAirflowCluster, - role: &str, - role_group: &str, -) -> Labels { - use crate::framework::types::operator::*; - framework::kvp::label::role_group_selector( - cluster, - &ProductName::from_str_unsafe(APP_NAME), - &RoleName::from_str_unsafe(role), - &RoleGroupName::from_str_unsafe(role_group), - ) -} - -fn prometheus_labels() -> Labels { - Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") -} - -fn prometheus_annotations() -> Annotations { - Annotations::try_from([ - ("prometheus.io/path".to_owned(), "/metrics".to_owned()), - ("prometheus.io/port".to_owned(), METRICS_PORT.to_string()), - ("prometheus.io/scheme".to_owned(), "http".to_owned()), - ("prometheus.io/scrape".to_owned(), "true".to_owned()), - ]) - .expect("should be valid annotations") -} - -fn build_group_listener( - cluster: &ValidatedAirflowCluster, - role: &AirflowRole, - listener_class: String, - listener_group_name: String, -) -> listener::v1alpha1::Listener { - let metadata = ObjectMetaBuilder::new() - .name(&listener_group_name) - .namespace(&cluster.namespace) - .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_labels(build_recommended_labels(cluster, &role.to_string(), "none")) - .build(); - - listener::v1alpha1::Listener { - metadata, - spec: listener::v1alpha1::ListenerSpec { - class_name: Some(listener_class), - ports: Some(listener_ports()), - ..listener::v1alpha1::ListenerSpec::default() - }, - status: None, - } -} - -fn listener_ports() -> Vec { - vec![listener::v1alpha1::ListenerPort { - name: HTTP_PORT_NAME.to_string(), - port: HTTP_PORT.into(), - protocol: Some("TCP".to_string()), - }] -} - -// --------------------------------------------------------------------------- -// RoleGroupBuilder -// --------------------------------------------------------------------------- - -struct RoleGroupBuilder<'a> { - cluster: &'a ValidatedAirflowCluster, - role_group_config: &'a ValidatedRoleGroupConfig, - rolegroup_ref: RoleGroupRef, - airflow_role: AirflowRole, - main_container: Container, - pod_data: &'a PrecomputedPodData, -} - -impl<'a> RoleGroupBuilder<'a> { - fn new( - cluster: &'a ValidatedAirflowCluster, - role_group_config: &'a ValidatedRoleGroupConfig, - rolegroup_ref: RoleGroupRef, - airflow_role: AirflowRole, - main_container: Container, - pod_data: &'a PrecomputedPodData, - ) -> Self { - Self { - cluster, - role_group_config, - rolegroup_ref, - airflow_role, - main_container, - pod_data, - } - } - - fn build_config_map(&self) -> ConfigMap { - let metadata = self - .common_metadata(self.rolegroup_ref.object_name()) - .build(); - - let mut cm_builder = ConfigMapBuilder::new(); - cm_builder.metadata(metadata); - - cm_builder.add_data( - AIRFLOW_CONFIG_FILENAME, - self.role_group_config.config_file_content.clone(), - ); - - extend_config_map_with_log_config( - &self.rolegroup_ref, - &self.main_container, - &self.role_group_config.logging.airflow_container, - self.role_group_config.logging.vector_container.as_ref(), - &mut cm_builder, - &self.cluster.image, - ); - - cm_builder - .build() - .expect("ConfigMap should build because metadata is set") - } - - fn build_stateful_set(&self) -> StatefulSet { - let restarter_label = Label::try_from(("restarter.stackable.tech/enabled", "true")) - .expect("static label is always valid"); - - let metadata = self - .common_metadata(self.rolegroup_ref.object_name()) - .with_label(restarter_label) - .build(); - - let template = self.build_pod_template(); - - let match_labels = { - use crate::framework::types::operator::*; - framework::kvp::label::role_group_selector( - self.cluster, - &ProductName::from_str_unsafe(APP_NAME), - &RoleName::from_str_unsafe(&self.rolegroup_ref.role), - &RoleGroupName::from_str_unsafe(&self.rolegroup_ref.role_group), - ) - }; - - let pod_management_policy = match self.airflow_role { - AirflowRole::Scheduler => "OrderedReady", - AirflowRole::Webserver - | AirflowRole::Worker - | AirflowRole::DagProcessor - | AirflowRole::Triggerer => "Parallel", - } - .to_string(); - - let spec = StatefulSetSpec { - pod_management_policy: Some(pod_management_policy), - replicas: self.pod_data.replicas.map(i32::from), - selector: LabelSelector { - match_labels: Some(match_labels.into()), - ..LabelSelector::default() - }, - service_name: stateful_set_service_name(&self.rolegroup_ref), - template, - volume_claim_templates: self - .pod_data - .listener_volume_claim_template - .clone() - .map(|pvc| vec![pvc]), - ..StatefulSetSpec::default() - }; - - StatefulSet { - metadata, - spec: Some(spec), - status: None, - } - } - - fn build_pod_template(&self) -> PodTemplateSpec { - let pod_metadata = ObjectMetaBuilder::new() - .with_labels(self.recommended_labels()) - .with_annotation( - Annotation::try_from(( - "kubectl.kubernetes.io/default-container", - format!("{}", self.main_container), - )) - .expect("static annotation is always valid"), - ) - .build(); - - let airflow_container = self.build_airflow_container(); - let metrics_container = self.build_metrics_container(); - - let mut containers = vec![airflow_container, metrics_container]; - containers.extend(self.pod_data.git_sync_containers.clone()); - if let Some(vector_container) = &self.pod_data.vector_container { - containers.push(vector_container.clone()); - } - - let init_containers = if self.pod_data.git_sync_init_containers.is_empty() { - None - } else { - Some(self.pod_data.git_sync_init_containers.clone()) - }; - - let volumes = self.build_volumes(); - - let termination_grace_period_seconds = self - .role_group_config - .graceful_shutdown_timeout - .as_secs() - .try_into() - .ok(); - - let mut pod_template = PodTemplateSpec { - metadata: Some(pod_metadata), - spec: Some(PodSpec { - affinity: { - let a = &self.role_group_config.affinity; - if a.pod_affinity.is_some() - || a.pod_anti_affinity.is_some() - || a.node_affinity.is_some() - { - Some(Affinity { - pod_affinity: a.pod_affinity.clone(), - pod_anti_affinity: a.pod_anti_affinity.clone(), - node_affinity: a.node_affinity.clone(), - }) - } else { - None - } - }, - containers, - init_containers, - service_account_name: Some(self.pod_data.service_account_name.clone()), - termination_grace_period_seconds, - security_context: Some(PodSecurityContext { - fs_group: Some(1000), - ..PodSecurityContext::default() - }), - image_pull_secrets: self.cluster.image.pull_secrets.clone(), - volumes: if volumes.is_empty() { - None - } else { - Some(volumes) - }, - ..PodSpec::default() - }), - }; - - pod_template.merge_from(self.pod_data.pod_overrides.clone()); - pod_template - } - - fn build_airflow_container(&self) -> K8sContainer { - let mut volume_mounts = vec![ - VolumeMount { - name: CONFIG_VOLUME_NAME.to_string(), - mount_path: CONFIG_PATH.to_string(), - ..VolumeMount::default() - }, - VolumeMount { - name: LOG_CONFIG_VOLUME_NAME.to_string(), - mount_path: LOG_CONFIG_DIR.to_string(), - ..VolumeMount::default() - }, - VolumeMount { - name: LOG_VOLUME_NAME.to_string(), - mount_path: STACKABLE_LOG_DIR.to_string(), - ..VolumeMount::default() - }, - ]; - - volume_mounts.extend(self.pod_data.extra_volume_mounts.clone()); - volume_mounts.extend(self.pod_data.auth_volume_mounts.clone()); - volume_mounts.extend(self.pod_data.git_sync_volume_mounts.clone()); - - if matches!( - self.pod_data.executor, - AirflowExecutor::KubernetesExecutors { .. } - ) { - volume_mounts.push(VolumeMount { - name: TEMPLATE_VOLUME_NAME.to_string(), - mount_path: TEMPLATE_LOCATION.to_string(), - ..VolumeMount::default() - }); - } - - if self.airflow_role.get_http_port().is_some() - && self.pod_data.listener_volume_claim_template.is_some() - { - volume_mounts.push(VolumeMount { - name: LISTENER_VOLUME_NAME.to_string(), - mount_path: LISTENER_VOLUME_DIR.to_string(), - ..VolumeMount::default() - }); - } - - let mut ports = Vec::new(); - if let Some(http_port) = self.airflow_role.get_http_port() { - ports.push(ContainerPort { - name: Some(HTTP_PORT_NAME.to_string()), - container_port: http_port.into(), - ..ContainerPort::default() - }); - } - - let (readiness_probe, liveness_probe) = - if let Some(http_port) = self.airflow_role.get_http_port() { - let probe = Probe { - tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(http_port.into()), - ..TCPSocketAction::default() - }), - initial_delay_seconds: Some(60), - period_seconds: Some(10), - failure_threshold: Some(6), - ..Probe::default() - }; - (Some(probe.clone()), Some(probe)) - } else { - (None, None) - }; - - K8sContainer { - name: self.main_container.to_string(), - image: Some(self.cluster.image.image.clone()), - image_pull_policy: Some(self.cluster.image.image_pull_policy.clone()), - command: Some(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]), - args: Some(vec![self.pod_data.airflow_commands.join("\n")]), - env: Some(self.pod_data.env_vars.clone()), - ports: if ports.is_empty() { None } else { Some(ports) }, - volume_mounts: Some(volume_mounts), - resources: Some(self.role_group_config.resources.clone().into()), - readiness_probe, - liveness_probe, - ..K8sContainer::default() - } - } - - fn build_metrics_container(&self) -> K8sContainer { - let args = [ - COMMON_BASH_TRAP_FUNCTIONS.to_string(), - "prepare_signal_handlers".to_string(), - "/stackable/statsd_exporter &".to_string(), - "wait_for_termination $!".to_string(), - ] - .join("\n"); - - K8sContainer { - name: "metrics".to_string(), - image: Some(self.cluster.image.image.clone()), - image_pull_policy: Some(self.cluster.image.image_pull_policy.clone()), - command: Some(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]), - args: Some(vec![args]), - ports: Some(vec![ContainerPort { - name: Some(METRICS_PORT_NAME.to_string()), - container_port: METRICS_PORT.into(), - ..ContainerPort::default() - }]), - resources: Some(ResourceRequirements { - requests: Some(BTreeMap::from([ - ("cpu".to_string(), Quantity("100m".to_string())), - ("memory".to_string(), Quantity("64Mi".to_string())), - ])), - limits: Some(BTreeMap::from([ - ("cpu".to_string(), Quantity("200m".to_string())), - ("memory".to_string(), Quantity("64Mi".to_string())), - ])), - ..ResourceRequirements::default() - }), - ..K8sContainer::default() - } - } - - fn build_volumes(&self) -> Vec { - // REVIEW: controller_commons::create_volumes is called with the new validated type - // (&ValidatedContainerLogConfigChoice) instead of the old Option<&ContainerLogConfig>. - // This is safe because the logging config has already been validated during the - // validate stage. - let mut volumes = controller_commons::create_volumes( - &self.rolegroup_ref.object_name(), - &self.role_group_config.logging.airflow_container, - ); - - volumes.extend(self.pod_data.extra_volumes.clone()); - volumes.extend(self.pod_data.auth_volumes.clone()); - volumes.extend(self.pod_data.git_sync_volumes.clone()); - - if let Some(template_cm_name) = &self.pod_data.executor_template_configmap_name { - volumes.push(Volume { - name: TEMPLATE_VOLUME_NAME.to_string(), - config_map: Some( - stackable_operator::k8s_openapi::api::core::v1::ConfigMapVolumeSource { - name: template_cm_name.clone(), - ..Default::default() - }, - ), - ..Volume::default() - }); - } - - volumes - } - - fn common_metadata(&self, resource_name: impl Into) -> ObjectMetaBuilder { - let mut builder = ObjectMetaBuilder::new(); - - builder - .name(resource_name) - .namespace(&self.cluster.namespace) - .ownerreference(ownerreference_from_resource(self.cluster, None, Some(true))) - .with_labels(self.recommended_labels()); - - builder - } - - fn recommended_labels(&self) -> Labels { - use crate::framework::types::operator::*; - framework::kvp::label::recommended_labels( - self.cluster, - &ProductName::from_str_unsafe(APP_NAME), - &ProductVersion::from_str_unsafe( - &self.cluster.image.app_version_label_value.to_string(), - ), - &OperatorName::from_str_unsafe(OPERATOR_NAME), - &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), - &RoleName::from_str_unsafe(&self.rolegroup_ref.role), - &RoleGroupName::from_str_unsafe(&self.rolegroup_ref.role_group), - ) - } -} - -// --------------------------------------------------------------------------- -// Apply -// --------------------------------------------------------------------------- - -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -pub enum ApplyError { - #[snafu(display("failed to apply resource"))] - ApplyResource { - source: stackable_operator::cluster_resources::Error, - }, - - #[snafu(display("failed to delete orphaned resources"))] - DeleteOrphanedResources { - source: stackable_operator::cluster_resources::Error, - }, -} - -struct Applier<'a> { - client: &'a stackable_operator::client::Client, - cluster_resources: ClusterResources<'a>, -} - -impl<'a> Applier<'a> { - fn new( - client: &'a stackable_operator::client::Client, - cluster_resources: ClusterResources<'a>, - ) -> Self { - Applier { - client, - cluster_resources, - } - } - - async fn apply( - mut self, - resources: KubernetesResources, - ) -> std::result::Result, ApplyError> { - let config_maps = self.add_resources(resources.config_maps).await?; - let service_accounts = self.add_resources(resources.service_accounts).await?; - let services = self.add_resources(resources.services).await?; - let role_bindings = self.add_resources(resources.role_bindings).await?; - let listeners = self.add_resources(resources.listeners).await?; - let stateful_sets = self.add_resources(resources.stateful_sets).await?; - let pod_disruption_budgets = self.add_resources(resources.pod_disruption_budgets).await?; - - self.cluster_resources - .delete_orphaned_resources(self.client) - .await - .context(apply_err::DeleteOrphanedResourcesSnafu)?; - - Ok(KubernetesResources { - stateful_sets, - config_maps, - services, - service_accounts, - role_bindings, - pod_disruption_budgets, - listeners, - _status: PhantomData, - }) - } - - async fn add_resources( - &mut self, - resources: Vec, - ) -> std::result::Result, ApplyError> { - let mut applied = vec![]; - for resource in resources { - let applied_resource = self - .cluster_resources - .add(self.client, resource) - .await - .context(apply_err::ApplyResourceSnafu)?; - applied.push(applied_resource); - } - Ok(applied) - } -} - -/// Module-like namespace for Snafu context selectors for ApplyError. -mod apply_err { - pub(super) use super::{ApplyResourceSnafu, DeleteOrphanedResourcesSnafu}; -} - -// --------------------------------------------------------------------------- -// Update status -// --------------------------------------------------------------------------- - -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -pub enum UpdateStatusError { - #[snafu(display("failed to update status"))] - PatchStatus { - source: stackable_operator::client::Error, - }, -} - -async fn update_status( - client: &stackable_operator::client::Client, - airflow: &v1alpha2::AirflowCluster, - applied_resources: KubernetesResources, -) -> std::result::Result<(), UpdateStatusError> { - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for stateful_set in applied_resources.stateful_sets { - ss_cond_builder.add(stateful_set); - } - - let cluster_operation_cond_builder = - ClusterOperationsConditionBuilder::new(&airflow.spec.cluster_operation); - - let status = AirflowClusterStatus { - conditions: compute_conditions( - airflow, - &[&ss_cond_builder, &cluster_operation_cond_builder], - ), - }; - - client - .apply_patch_status(OPERATOR_NAME, airflow, &status) - .await - .context(PatchStatusSnafu)?; - - Ok(()) -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use std::{collections::BTreeMap, str::FromStr}; - - use stackable_operator::{ - commons::{ - affinity::StackableAffinity, - product_image_selection::ResolvedProductImage, - resources::{NoRuntimeLimits, Resources}, - }, - k8s_openapi::api::core::v1::PodTemplateSpec, - kube::Resource, - kvp::LabelValue, - product_logging::spec::{ - AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging, - }, - shared::time::Duration, - }; - - use super::*; - use crate::crd::{AirflowConfig, AirflowStorageConfig, Container}; - - // --- validate tests --- - - fn airflow_config_with_logging(enable_vector: bool) -> AirflowConfig { - let mut containers = BTreeMap::new(); - containers.insert( - Container::Airflow, - ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - )), - }, - ); - if enable_vector { - containers.insert( - Container::Vector, - ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - )), - }, - ); - } - AirflowConfig { - resources: Default::default(), - logging: Logging { - enable_vector_agent: enable_vector, - containers, - }, - affinity: Default::default(), - graceful_shutdown_timeout: Some(Duration::from_secs(120)), - } - } - - #[test] - fn test_validate_airflow_config_without_vector() { - let config = airflow_config_with_logging(false); - let result = validate_airflow_config(&config, None, String::new()); - assert!(result.is_ok()); - let validated = result.unwrap(); - assert!(validated.logging.vector_container.is_none()); - assert!(!validated.logging.is_vector_agent_enabled()); - assert_eq!( - validated.graceful_shutdown_timeout, - Duration::from_secs(120) - ); - } - - #[test] - fn test_validate_airflow_config_with_vector() { - let config = airflow_config_with_logging(true); - let result = - validate_airflow_config(&config, Some("vector-aggregator-discovery"), String::new()); - assert!(result.is_ok()); - let validated = result.unwrap(); - assert!(validated.logging.vector_container.is_some()); - assert!(validated.logging.is_vector_agent_enabled()); - } - - #[test] - fn test_validate_vector_enabled_missing_config_map() { - let config = airflow_config_with_logging(true); - let result = validate_airflow_config(&config, None, String::new()); - assert!(result.is_err()); - } - - #[test] - fn test_validate_missing_graceful_shutdown() { - let mut config = airflow_config_with_logging(false); - config.graceful_shutdown_timeout = None; - let result = validate_airflow_config(&config, None, String::new()); - assert!(result.is_err()); - } - - #[test] - fn test_validate_ok() { - let (airflow, image) = test_objects(); - let result = validate_and_assemble( - &airflow, - &image, - BTreeMap::new(), - BTreeMap::new(), - vec![], - BTreeMap::new(), - ); - assert!(result.is_ok()); - let validated = result.unwrap(); - assert_eq!(validated.name.to_string(), "my-airflow"); - assert_eq!(validated.namespace.to_string(), "default"); - } - - #[test] - fn test_validate_err_missing_name() { - test_validate_err( - |airflow, _| airflow.metadata.name = None, - ValidateErrorDiscriminants::InvalidClusterName, - ); - } - - #[test] - fn test_validate_err_missing_namespace() { - test_validate_err( - |airflow, _| airflow.metadata.namespace = None, - ValidateErrorDiscriminants::ObjectHasNoNamespace, - ); - } - - #[test] - fn test_validate_err_missing_uid() { - test_validate_err( - |airflow, _| airflow.metadata.uid = None, - ValidateErrorDiscriminants::ObjectHasNoUid, - ); - } - - #[test] - fn test_validate_err_invalid_cluster_name() { - test_validate_err( - |airflow, _| { - airflow.metadata.name = - Some("THIS-IS-NOT-A-VALID-DNS-LABEL-NAME-BECAUSE-UPPERCASE".to_string()) - }, - ValidateErrorDiscriminants::InvalidClusterName, - ); - } - - #[test] - fn test_validate_err_invalid_namespace() { - test_validate_err( - |airflow, _| airflow.metadata.namespace = Some("INVALID NAMESPACE".to_string()), - ValidateErrorDiscriminants::InvalidClusterNamespace, - ); - } - - #[test] - fn test_validate_err_invalid_uid() { - test_validate_err( - |airflow, _| airflow.metadata.uid = Some("not-a-uuid".to_string()), - ValidateErrorDiscriminants::InvalidClusterUid, - ); - } - - fn test_validate_err( - mutate: fn(&mut v1alpha2::AirflowCluster, &mut ResolvedProductImage), - expected: ValidateErrorDiscriminants, - ) { - let (mut airflow, mut image) = test_objects(); - mutate(&mut airflow, &mut image); - let result = validate_and_assemble( - &airflow, - &image, - BTreeMap::new(), - BTreeMap::new(), - vec![], - BTreeMap::new(), - ); - match result { - Err(err) => assert_eq!(expected, ValidateErrorDiscriminants::from(err)), - Ok(_) => panic!("validate should have failed with {expected:?}"), - } - } - - fn test_objects() -> (v1alpha2::AirflowCluster, ResolvedProductImage) { - let airflow = v1alpha2::AirflowCluster { - metadata: ObjectMeta { - name: Some("my-airflow".to_string()), - namespace: Some("default".to_string()), - uid: Some("e6ac237d-a6d4-43a1-8135-f36506110912".to_string()), - ..ObjectMeta::default() - }, - spec: serde_json::from_value(serde_json::json!({ - "image": { "productVersion": "2.10.4" }, - "clusterConfig": { - "credentialsSecretName": "airflow-admin-credentials", - "metadataDatabase": { - "postgresql": { - "host": "airflow-postgresql", - "database": "airflow", - "credentialsSecretName": "airflow-postgresql-credentials" - } - } - }, - "kubernetesExecutors": { "config": {} }, - "webservers": { "roleGroups": { "default": { "config": {} } } }, - "schedulers": { "roleGroups": { "default": { "config": {} } } } - })) - .expect("test spec JSON should be valid"), - status: None, - }; - - let image = ResolvedProductImage { - product_version: "2.10.4".to_owned(), - app_version_label_value: LabelValue::from_str("2.10.4-stackable0.0.0-dev") - .expect("valid label value"), - image: "oci.stackable.tech/sdp/airflow:2.10.4-stackable0.0.0-dev".to_string(), - image_pull_policy: "Always".to_owned(), - pull_secrets: None, - }; - - (airflow, image) - } - - // --- build tests --- - - #[test] - fn test_build() { - let validated = validated_cluster(); - - let resources = build(&validated); - - assert_eq!( - vec![ - "my-airflow-scheduler-default", - "my-airflow-webserver-default", - ], - extract_resource_names(&resources.stateful_sets) - ); - assert_eq!( - vec![ - "my-airflow-scheduler-default-headless", - "my-airflow-scheduler-default-metrics", - "my-airflow-webserver-default-headless", - "my-airflow-webserver-default-metrics", - ], - extract_resource_names(&resources.services) - ); - assert_eq!( - vec![ - "my-airflow-scheduler-default", - "my-airflow-webserver-default", - ], - extract_resource_names(&resources.config_maps) - ); - assert_eq!( - vec!["my-airflow-serviceaccount"], - extract_resource_names(&resources.service_accounts) - ); - assert_eq!( - vec!["my-airflow-rolebinding"], - extract_resource_names(&resources.role_bindings) - ); - assert_eq!( - vec!["my-airflow-scheduler", "my-airflow-webserver"], - extract_resource_names(&resources.pod_disruption_budgets) - ); - assert_eq!( - vec!["my-airflow-webserver"], - extract_resource_names(&resources.listeners) - ); - } - - fn extract_resource_names(resources: &[impl Resource]) -> Vec<&str> { - let mut names: Vec<&str> = resources - .iter() - .filter_map(|r| r.meta().name.as_ref()) - .map(|n| n.as_str()) - .collect(); - names.sort(); - names - } - - fn validated_cluster() -> ValidatedAirflowCluster { - let image = ResolvedProductImage { - product_version: "2.10.4".to_owned(), - app_version_label_value: LabelValue::from_str("2.10.4-stackable0.0.0-dev") - .expect("valid label value"), - image: "oci.stackable.tech/sdp/airflow:2.10.4-stackable0.0.0-dev".to_string(), - image_pull_policy: "Always".to_owned(), - pull_secrets: None, - }; - - let logging = ValidatedLogging { - airflow_container: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), - vector_container: None, - git_sync_container_log_config: ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - )), - }, - }; - - let role_group_config = ValidatedRoleGroupConfig { - resources: Resources::::default(), - logging: logging.clone(), - affinity: StackableAffinity::default(), - graceful_shutdown_timeout: Duration::from_secs(120), - config_file_content: String::new(), - }; - - let pod_data = PrecomputedPodData { - env_vars: vec![], - airflow_commands: vec!["airflow webserver".to_string()], - auth_volumes: vec![], - auth_volume_mounts: vec![], - extra_volumes: vec![], - extra_volume_mounts: vec![], - git_sync_containers: vec![], - git_sync_init_containers: vec![], - git_sync_volumes: vec![], - git_sync_volume_mounts: vec![], - vector_container: None, - service_account_name: "my-airflow-serviceaccount".to_string(), - replicas: Some(1), - pod_overrides: PodTemplateSpec::default(), - executor: AirflowExecutor::KubernetesExecutors { - common_configuration: Box::default(), - }, - executor_template_configmap_name: None, - listener_volume_claim_template: None, - }; - - let role_groups = BTreeMap::from([ - ( - AirflowRole::Webserver, - BTreeMap::from([("default".to_string(), role_group_config.clone())]), - ), - ( - AirflowRole::Scheduler, - BTreeMap::from([("default".to_string(), role_group_config)]), - ), - ]); - - let precomputed_pod_data = BTreeMap::from([ - ( - AirflowRole::Webserver, - BTreeMap::from([("default".to_string(), pod_data.clone())]), - ), - ( - AirflowRole::Scheduler, - BTreeMap::from([("default".to_string(), pod_data)]), - ), - ]); - - // Role configs: PDB enabled for both roles; Webserver also gets a listener - let role_configs = BTreeMap::from([ - ( - AirflowRole::Scheduler, - ValidatedRoleConfig { - pdb_enabled: true, - pdb_max_unavailable: None, - listener_class: None, - group_listener_name: None, - }, - ), - ( - AirflowRole::Webserver, - ValidatedRoleConfig { - pdb_enabled: true, - pdb_max_unavailable: None, - listener_class: Some("cluster-internal".to_string()), - group_listener_name: Some("my-airflow-webserver".to_string()), - }, - ), - ]); - - ValidatedAirflowCluster::new( - image, - ClusterName::from_str_unsafe("my-airflow"), - NamespaceName::from_str_unsafe("default"), - Uid::from_str_unsafe("e6ac237d-a6d4-43a1-8135-f36506110912"), - role_groups, - precomputed_pod_data, - vec![], - role_configs, - AirflowExecutor::KubernetesExecutors { - common_configuration: Box::default(), - }, - ) - } -} diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs new file mode 100644 index 00000000..8ed7a1de --- /dev/null +++ b/rust/operator-binary/src/controller.rs @@ -0,0 +1,378 @@ +//! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`] +//! +//! Pipeline architecture: dereference -> validate -> build -> apply -> update_status + +pub mod apply; +pub mod build; +pub mod dereference; +pub mod update_status; +pub mod validate; + +// --------------------------------------------------------------------------- +// Imports +// --------------------------------------------------------------------------- + +use std::{ + collections::BTreeMap, + marker::PhantomData, + sync::Arc, +}; + +use const_format::concatcp; +use product_config::ProductConfigManager; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + cli::OperatorEnvironmentOptions, + cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, + commons::{ + affinity::StackableAffinity, + product_image_selection::ResolvedProductImage, + resources::{NoRuntimeLimits, Resources}, + }, + crd::listener, + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ + ConfigMap, Container as K8sContainer, EnvVar, PersistentVolumeClaim, PodTemplateSpec, + Service, ServiceAccount, Volume, VolumeMount, + }, + policy::v1::PodDisruptionBudget, + rbac::v1::RoleBinding, + }, + kube::{ + Resource, + api::ObjectMeta, + core::{DeserializeGuard, error_boundary}, + runtime::{controller::Action, reflector::ObjectRef}, + }, + logging::controller::ReconcilerError, + product_logging::spec::ContainerLogConfig, + role_utils::RoleGroupRef, + shared::time::Duration, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::{ + crd::{ + APP_NAME, AirflowExecutor, AirflowRole, AirflowStorageConfig, OPERATOR_NAME, + v1alpha2, + }, + framework::{ + HasName, HasUid, NameIsValidLabelValue, + product_logging::framework::{ValidatedContainerLogConfigChoice, VectorContainerLogConfig}, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, +}; + +use self::{ + apply::Applier, + build::build, + dereference::dereference, + update_status::update_status, + validate::validate_cluster, +}; + +// --------------------------------------------------------------------------- +// Constants and context +// --------------------------------------------------------------------------- + +pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; +pub const CONTAINER_IMAGE_BASE_NAME: &str = "airflow"; +pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = + concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); + +pub struct Ctx { + pub client: stackable_operator::client::Client, + pub product_config: ProductConfigManager, + pub operator_environment: OperatorEnvironmentOptions, +} + +// --------------------------------------------------------------------------- +// Validated types +// --------------------------------------------------------------------------- + +pub(crate) struct Prepared; +pub(crate) struct Applied; + +pub(crate) struct KubernetesResources { + pub stateful_sets: Vec, + pub config_maps: Vec, + pub services: Vec, + pub service_accounts: Vec, + pub role_bindings: Vec, + pub pod_disruption_budgets: Vec, + pub listeners: Vec, + pub _status: PhantomData, +} + +#[derive(Clone, Debug)] +pub struct ValidatedRoleConfig { + pub pdb_enabled: bool, + pub pdb_max_unavailable: Option, + pub listener_class: Option, + pub group_listener_name: Option, +} + +#[derive(Clone, Debug)] +pub struct ValidatedRoleGroupConfig { + pub resources: Resources, + pub logging: ValidatedLogging, + pub affinity: StackableAffinity, + pub graceful_shutdown_timeout: Duration, + pub config_file_content: String, +} + +#[derive(Clone)] +pub struct PrecomputedPodData { + pub env_vars: Vec, + pub airflow_commands: Vec, + pub auth_volumes: Vec, + pub auth_volume_mounts: Vec, + pub extra_volumes: Vec, + pub extra_volume_mounts: Vec, + pub git_sync_containers: Vec, + pub git_sync_init_containers: Vec, + pub git_sync_volumes: Vec, + pub git_sync_volume_mounts: Vec, + pub vector_container: Option, + pub service_account_name: String, + pub replicas: Option, + pub pod_overrides: PodTemplateSpec, + pub executor: AirflowExecutor, + pub executor_template_configmap_name: Option, + pub listener_volume_claim_template: Option, +} + +#[derive(Clone, Debug)] +pub struct ValidatedLogging { + pub airflow_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, + pub git_sync_container_log_config: ContainerLogConfig, +} + +impl ValidatedLogging { + pub fn is_vector_agent_enabled(&self) -> bool { + self.vector_container.is_some() + } +} + +// REVIEW: ValidatedAirflowCluster is the central validated type. It holds all data needed +// by the build stage so that build() can be infallible. All optional-after-merge fields +// are unwrapped during validation, and logging is pre-validated into ValidatedLogging. +#[derive(Clone)] +pub struct ValidatedAirflowCluster { + metadata: ObjectMeta, + pub image: ResolvedProductImage, + pub name: ClusterName, + pub namespace: NamespaceName, + pub uid: Uid, + pub role_groups: BTreeMap>, + pub precomputed_pod_data: BTreeMap>, + pub executor_template_config_maps: Vec, + pub role_configs: BTreeMap, + pub executor: AirflowExecutor, +} + +impl ValidatedAirflowCluster { + #[allow(clippy::too_many_arguments)] + pub fn new( + image: ResolvedProductImage, + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + role_groups: BTreeMap>, + precomputed_pod_data: BTreeMap>, + executor_template_config_maps: Vec, + role_configs: BTreeMap, + executor: AirflowExecutor, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + image, + name, + namespace, + uid, + role_groups, + precomputed_pod_data, + executor_template_config_maps, + role_configs, + executor, + } + } + + pub fn rolegroup_ref(&self, role: &AirflowRole, role_group: &str) -> RoleGroupRef { + RoleGroupRef { + cluster: ObjectRef::from_obj(self), + role: role.to_string(), + role_group: role_group.to_string(), + } + } +} + +impl HasName for ValidatedAirflowCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedAirflowCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + +impl Resource for ValidatedAirflowCluster { + type DynamicType = + ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha2::AirflowCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +impl NameIsValidLabelValue for ValidatedAirflowCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } +} + +// --------------------------------------------------------------------------- +// Error types and reconcile +// --------------------------------------------------------------------------- + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("AirflowCluster object is invalid"))] + InvalidAirflowCluster { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to dereference resources"))] + Dereference { source: dereference::Error }, + + #[snafu(display("failed to validate cluster"))] + Validate { source: validate::Error }, + + #[snafu(display("failed to create cluster resources"))] + CreateClusterResources { + source: stackable_operator::cluster_resources::Error, + }, + + #[snafu(display("failed to apply resources"))] + Apply { source: apply::Error }, + + #[snafu(display("failed to update status"))] + UpdateStatus { source: update_status::Error }, +} + +type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +// REVIEW: The reconcile pipeline is structured as five sequential stages: +// 1. dereference — async, fallible: resolve external references +// 2. validate — sync, fallible: validate and merge configs +// 3. build — sync, infallible: construct Kubernetes resources +// 4. apply — async, fallible: apply resources to the cluster +// 5. update_status — async, fallible: patch status on the CRD +pub async fn reconcile( + airflow: Arc>, + ctx: Arc, +) -> Result { + tracing::info!("Starting reconcile"); + + let airflow = airflow + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidAirflowClusterSnafu)?; + + // --- dereference (async, fallible) --- + let dereferenced = dereference( + &ctx.client, + airflow, + CONTAINER_IMAGE_BASE_NAME, + &ctx.operator_environment.image_repository, + crate::built_info::PKG_VERSION, + ) + .await + .context(DereferenceSnafu)?; + + // --- validate (sync, fallible) --- + let validated = + validate_cluster(airflow, &dereferenced, &ctx.product_config).context(ValidateSnafu)?; + + // REVIEW: build() is infallible — all validation and fallible operations (config + // generation, PodBuilder/ContainerBuilder usage, logging validation) happen in the + // validate stage. The build stage purely assembles Kubernetes resource structs. + // --- build (sync, infallible) --- + let prepared = build(&validated); + + // --- apply (async, fallible) --- + let cluster_resources = ClusterResources::new( + APP_NAME, + OPERATOR_NAME, + AIRFLOW_CONTROLLER_NAME, + &airflow.object_ref(&()), + ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), + &airflow.spec.object_overrides, + ) + .context(CreateClusterResourcesSnafu)?; + + let applied = Applier::new(&ctx.client, cluster_resources) + .apply(prepared) + .await + .context(ApplySnafu)?; + + // --- update status (async, fallible) --- + update_status(&ctx.client, airflow, applied) + .await + .context(UpdateStatusSnafu)?; + + Ok(Action::await_change()) +} + +pub fn error_policy( + _obj: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidAirflowCluster { .. } => Action::await_change(), + _ => Action::requeue(*Duration::from_secs(10)), + } +} diff --git a/rust/operator-binary/src/controller/apply.rs b/rust/operator-binary/src/controller/apply.rs new file mode 100644 index 00000000..e5eb21a8 --- /dev/null +++ b/rust/operator-binary/src/controller/apply.rs @@ -0,0 +1,83 @@ +use std::marker::PhantomData; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::cluster_resources::{ClusterResource, ClusterResources}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use super::{Applied, KubernetesResources, Prepared}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to apply resource"))] + ApplyResource { + source: stackable_operator::cluster_resources::Error, + }, + + #[snafu(display("failed to delete orphaned resources"))] + DeleteOrphanedResources { + source: stackable_operator::cluster_resources::Error, + }, +} + +pub(crate) struct Applier<'a> { + client: &'a stackable_operator::client::Client, + cluster_resources: ClusterResources<'a>, +} + +impl<'a> Applier<'a> { + pub(crate) fn new( + client: &'a stackable_operator::client::Client, + cluster_resources: ClusterResources<'a>, + ) -> Self { + Applier { + client, + cluster_resources, + } + } + + pub(crate) async fn apply( + mut self, + resources: KubernetesResources, + ) -> std::result::Result, Error> { + let config_maps = self.add_resources(resources.config_maps).await?; + let service_accounts = self.add_resources(resources.service_accounts).await?; + let services = self.add_resources(resources.services).await?; + let role_bindings = self.add_resources(resources.role_bindings).await?; + let listeners = self.add_resources(resources.listeners).await?; + let stateful_sets = self.add_resources(resources.stateful_sets).await?; + let pod_disruption_budgets = self.add_resources(resources.pod_disruption_budgets).await?; + + self.cluster_resources + .delete_orphaned_resources(self.client) + .await + .context(DeleteOrphanedResourcesSnafu)?; + + Ok(KubernetesResources { + stateful_sets, + config_maps, + services, + service_accounts, + role_bindings, + pod_disruption_budgets, + listeners, + _status: PhantomData, + }) + } + + async fn add_resources( + &mut self, + resources: Vec, + ) -> std::result::Result, Error> { + let mut applied = vec![]; + for resource in resources { + let applied_resource = self + .cluster_resources + .add(self.client, resource) + .await + .context(ApplyResourceSnafu)?; + applied.push(applied_resource); + } + Ok(applied) + } +} diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs new file mode 100644 index 00000000..952d9605 --- /dev/null +++ b/rust/operator-binary/src/controller/build.rs @@ -0,0 +1,529 @@ +pub mod role_group_builder; + +use std::marker::PhantomData; + +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + commons::rbac::build_rbac_resources, + crd::listener, + k8s_openapi::api::core::v1::{ServicePort, ServiceSpec}, + kvp::{Annotations, Labels}, + role_utils::RoleGroupRef, +}; + +use crate::{ + crd::{ + APP_NAME, AirflowExecutor, AirflowRole, Container, HTTP_PORT, HTTP_PORT_NAME, + METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, + }, + framework::{ + self, + builder::meta::ownerreference_from_resource, + types::operator::*, + }, +}; + +use super::{ + AIRFLOW_CONTROLLER_NAME, KubernetesResources, Prepared, ValidatedAirflowCluster, + ValidatedRoleConfig, +}; +use self::role_group_builder::RoleGroupBuilder; + +fn main_container_for_role(_role: &AirflowRole) -> Container { + Container::Airflow +} + +// REVIEW: build() is infallible. All validation and fallible operations (config generation, +// PodBuilder/ContainerBuilder usage, logging validation) are performed in the validate +// stage. The build stage purely assembles Kubernetes resource structs from validated data. +pub(crate) fn build(validated: &ValidatedAirflowCluster) -> KubernetesResources { + let mut stateful_sets = Vec::new(); + let mut config_maps = Vec::new(); + let mut services = Vec::new(); + let mut pod_disruption_budgets = Vec::new(); + let mut listeners = Vec::new(); + + // --- RBAC --- + let rbac_labels = build_recommended_labels(validated, "rbac", "rbac"); + + let (rbac_sa, rbac_rolebinding) = build_rbac_resources(validated, APP_NAME, rbac_labels) + .expect( + "RBAC resources should be created because the validated cluster has valid metadata", + ); + + // --- Executor template ConfigMaps (pre-built in validate stage) --- + config_maps.extend(validated.executor_template_config_maps.clone()); + + // --- Per-role/rolegroup resources --- + for (airflow_role, role_groups) in &validated.role_groups { + // PDBs + if let Some(role_config) = validated.role_configs.get(airflow_role) { + if let Some(pdb) = build_pdb(validated, airflow_role, role_config) { + pod_disruption_budgets.push(pdb); + } + } + + // Group listeners (only Webserver) + if let Some(role_config) = validated.role_configs.get(airflow_role) { + if let (Some(listener_class), Some(listener_name)) = ( + &role_config.listener_class, + &role_config.group_listener_name, + ) { + listeners.push(build_group_listener( + validated, + airflow_role, + listener_class.clone(), + listener_name.clone(), + )); + } + } + + for (rolegroup_name, role_group_config) in role_groups { + let rolegroup_ref = validated.rolegroup_ref(airflow_role, rolegroup_name); + + let main_container = main_container_for_role(airflow_role); + + // Services + services.push(build_headless_service(validated, &rolegroup_ref)); + services.push(build_metrics_service(validated, &rolegroup_ref)); + + // ConfigMap + StatefulSet via RoleGroupBuilder + let pod_data = validated + .precomputed_pod_data + .get(airflow_role) + .and_then(|groups| groups.get(rolegroup_name)) + .expect( + "PrecomputedPodData should exist for every role group \ + because validate_cluster computes it for each one", + ); + + let builder = RoleGroupBuilder::new( + validated, + role_group_config, + rolegroup_ref, + airflow_role.clone(), + main_container, + pod_data, + ); + + config_maps.push(builder.build_config_map()); + stateful_sets.push(builder.build_stateful_set()); + } + } + + KubernetesResources { + stateful_sets, + config_maps, + services, + service_accounts: vec![rbac_sa], + role_bindings: vec![rbac_rolebinding], + pod_disruption_budgets, + listeners, + _status: PhantomData, + } +} + +fn build_pdb( + cluster: &ValidatedAirflowCluster, + role: &AirflowRole, + role_config: &ValidatedRoleConfig, +) -> Option { + if !role_config.pdb_enabled { + return None; + } + + let max_unavailable = role_config.pdb_max_unavailable.unwrap_or(match role { + AirflowRole::Worker => match &cluster.executor { + AirflowExecutor::KubernetesExecutors { .. } => return None, + _ => 1, + }, + _ => 1, + }); + + // REVIEW: from_str_unsafe is used here because the values come from constants (APP_NAME, + // OPERATOR_NAME, AIRFLOW_CONTROLLER_NAME) or validated role names — they are known to be + // valid at compile time or have been validated during the validate stage. + Some({ + framework::builder::pdb::pod_disruption_budget_builder_with_role( + cluster, + &ProductName::from_str_unsafe(APP_NAME), + &RoleName::from_str_unsafe(&role.to_string()), + &OperatorName::from_str_unsafe(OPERATOR_NAME), + &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), + ) + .with_max_unavailable(max_unavailable) + .build() + }) +} + +fn build_headless_service( + cluster: &ValidatedAirflowCluster, + rolegroup_ref: &RoleGroupRef, +) -> stackable_operator::k8s_openapi::api::core::v1::Service { + let metadata = ObjectMetaBuilder::new() + .name(format!("{}-headless", rolegroup_ref.object_name())) + .namespace(&cluster.namespace) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(build_recommended_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .build(); + + stackable_operator::k8s_openapi::api::core::v1::Service { + metadata, + spec: Some(ServiceSpec { + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(vec![ServicePort { + name: Some(HTTP_PORT_NAME.to_string()), + port: HTTP_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }]), + selector: Some( + build_role_group_selector_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +fn build_metrics_service( + cluster: &ValidatedAirflowCluster, + rolegroup_ref: &RoleGroupRef, +) -> stackable_operator::k8s_openapi::api::core::v1::Service { + let metadata = ObjectMetaBuilder::new() + .name(format!("{}-metrics", rolegroup_ref.object_name())) + .namespace(&cluster.namespace) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(build_recommended_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .with_labels(prometheus_labels()) + .with_annotations(prometheus_annotations()) + .build(); + + stackable_operator::k8s_openapi::api::core::v1::Service { + metadata, + spec: Some(ServiceSpec { + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }]), + selector: Some( + build_role_group_selector_labels( + cluster, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +// REVIEW: from_str_unsafe is used for label construction throughout these helpers because +// the inputs are either compile-time constants (APP_NAME, OPERATOR_NAME, etc.) or values +// that have already been validated during the validate stage (role names, role group names, +// product versions). Using from_str_unsafe avoids redundant re-validation in the infallible +// build stage. +pub(super) fn build_recommended_labels( + cluster: &ValidatedAirflowCluster, + role: &str, + role_group: &str, +) -> Labels { + framework::kvp::label::recommended_labels( + cluster, + &ProductName::from_str_unsafe(APP_NAME), + &ProductVersion::from_str_unsafe(&cluster.image.app_version_label_value.to_string()), + &OperatorName::from_str_unsafe(OPERATOR_NAME), + &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), + &RoleName::from_str_unsafe(role), + &RoleGroupName::from_str_unsafe(role_group), + ) +} + +pub(super) fn build_role_group_selector_labels( + cluster: &ValidatedAirflowCluster, + role: &str, + role_group: &str, +) -> Labels { + framework::kvp::label::role_group_selector( + cluster, + &ProductName::from_str_unsafe(APP_NAME), + &RoleName::from_str_unsafe(role), + &RoleGroupName::from_str_unsafe(role_group), + ) +} + +fn prometheus_labels() -> Labels { + Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") +} + +fn prometheus_annotations() -> Annotations { + Annotations::try_from([ + ("prometheus.io/path".to_owned(), "/metrics".to_owned()), + ("prometheus.io/port".to_owned(), METRICS_PORT.to_string()), + ("prometheus.io/scheme".to_owned(), "http".to_owned()), + ("prometheus.io/scrape".to_owned(), "true".to_owned()), + ]) + .expect("should be valid annotations") +} + +fn build_group_listener( + cluster: &ValidatedAirflowCluster, + role: &AirflowRole, + listener_class: String, + listener_group_name: String, +) -> listener::v1alpha1::Listener { + let metadata = ObjectMetaBuilder::new() + .name(&listener_group_name) + .namespace(&cluster.namespace) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(build_recommended_labels(cluster, &role.to_string(), "none")) + .build(); + + listener::v1alpha1::Listener { + metadata, + spec: listener::v1alpha1::ListenerSpec { + class_name: Some(listener_class), + ports: Some(listener_ports()), + ..listener::v1alpha1::ListenerSpec::default() + }, + status: None, + } +} + +fn listener_ports() -> Vec { + vec![listener::v1alpha1::ListenerPort { + name: HTTP_PORT_NAME.to_string(), + port: HTTP_PORT.into(), + protocol: Some("TCP".to_string()), + }] +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_operator::{ + commons::{ + affinity::StackableAffinity, + product_image_selection::ResolvedProductImage, + resources::{NoRuntimeLimits, Resources}, + }, + k8s_openapi::api::core::v1::PodTemplateSpec, + kube::Resource, + kvp::LabelValue, + product_logging::spec::{ + AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + }, + shared::time::Duration, + }; + + use super::*; + use crate::{ + controller::{ + PrecomputedPodData, ValidatedAirflowCluster, ValidatedLogging, ValidatedRoleConfig, + ValidatedRoleGroupConfig, + }, + crd::{AirflowExecutor, AirflowRole, AirflowStorageConfig}, + framework::{ + product_logging::framework::ValidatedContainerLogConfigChoice, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, + }; + + #[test] + fn test_build() { + let validated = validated_cluster(); + + let resources = build(&validated); + + assert_eq!( + vec![ + "my-airflow-scheduler-default", + "my-airflow-webserver-default", + ], + extract_resource_names(&resources.stateful_sets) + ); + assert_eq!( + vec![ + "my-airflow-scheduler-default-headless", + "my-airflow-scheduler-default-metrics", + "my-airflow-webserver-default-headless", + "my-airflow-webserver-default-metrics", + ], + extract_resource_names(&resources.services) + ); + assert_eq!( + vec![ + "my-airflow-scheduler-default", + "my-airflow-webserver-default", + ], + extract_resource_names(&resources.config_maps) + ); + assert_eq!( + vec!["my-airflow-serviceaccount"], + extract_resource_names(&resources.service_accounts) + ); + assert_eq!( + vec!["my-airflow-rolebinding"], + extract_resource_names(&resources.role_bindings) + ); + assert_eq!( + vec!["my-airflow-scheduler", "my-airflow-webserver"], + extract_resource_names(&resources.pod_disruption_budgets) + ); + assert_eq!( + vec!["my-airflow-webserver"], + extract_resource_names(&resources.listeners) + ); + } + + fn extract_resource_names(resources: &[impl Resource]) -> Vec<&str> { + let mut names: Vec<&str> = resources + .iter() + .filter_map(|r| r.meta().name.as_ref()) + .map(|n| n.as_str()) + .collect(); + names.sort(); + names + } + + fn validated_cluster() -> ValidatedAirflowCluster { + let image = ResolvedProductImage { + product_version: "2.10.4".to_owned(), + app_version_label_value: LabelValue::from_str("2.10.4-stackable0.0.0-dev") + .expect("valid label value"), + image: "oci.stackable.tech/sdp/airflow:2.10.4-stackable0.0.0-dev".to_string(), + image_pull_policy: "Always".to_owned(), + pull_secrets: None, + }; + + let logging = ValidatedLogging { + airflow_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + git_sync_container_log_config: ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + }; + + let role_group_config = ValidatedRoleGroupConfig { + resources: Resources::::default(), + logging: logging.clone(), + affinity: StackableAffinity::default(), + graceful_shutdown_timeout: Duration::from_secs(120), + config_file_content: String::new(), + }; + + let pod_data = PrecomputedPodData { + env_vars: vec![], + airflow_commands: vec!["airflow webserver".to_string()], + auth_volumes: vec![], + auth_volume_mounts: vec![], + extra_volumes: vec![], + extra_volume_mounts: vec![], + git_sync_containers: vec![], + git_sync_init_containers: vec![], + git_sync_volumes: vec![], + git_sync_volume_mounts: vec![], + vector_container: None, + service_account_name: "my-airflow-serviceaccount".to_string(), + replicas: Some(1), + pod_overrides: PodTemplateSpec::default(), + executor: AirflowExecutor::KubernetesExecutors { + common_configuration: Box::default(), + }, + executor_template_configmap_name: None, + listener_volume_claim_template: None, + }; + + let role_groups = BTreeMap::from([ + ( + AirflowRole::Webserver, + BTreeMap::from([("default".to_string(), role_group_config.clone())]), + ), + ( + AirflowRole::Scheduler, + BTreeMap::from([("default".to_string(), role_group_config)]), + ), + ]); + + let precomputed_pod_data = BTreeMap::from([ + ( + AirflowRole::Webserver, + BTreeMap::from([("default".to_string(), pod_data.clone())]), + ), + ( + AirflowRole::Scheduler, + BTreeMap::from([("default".to_string(), pod_data)]), + ), + ]); + + // Role configs: PDB enabled for both roles; Webserver also gets a listener + let role_configs = BTreeMap::from([ + ( + AirflowRole::Scheduler, + ValidatedRoleConfig { + pdb_enabled: true, + pdb_max_unavailable: None, + listener_class: None, + group_listener_name: None, + }, + ), + ( + AirflowRole::Webserver, + ValidatedRoleConfig { + pdb_enabled: true, + pdb_max_unavailable: None, + listener_class: Some("cluster-internal".to_string()), + group_listener_name: Some("my-airflow-webserver".to_string()), + }, + ), + ]); + + ValidatedAirflowCluster::new( + image, + ClusterName::from_str_unsafe("my-airflow"), + NamespaceName::from_str_unsafe("default"), + Uid::from_str_unsafe("e6ac237d-a6d4-43a1-8135-f36506110912"), + role_groups, + precomputed_pod_data, + vec![], + role_configs, + AirflowExecutor::KubernetesExecutors { + common_configuration: Box::default(), + }, + ) + } +} diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs new file mode 100644 index 00000000..8e52744e --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -0,0 +1,416 @@ +use std::collections::BTreeMap; + +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + k8s_openapi::api::{ + apps::v1::{StatefulSet, StatefulSetSpec}, + core::v1::{ + Affinity, ConfigMap, Container as K8sContainer, ContainerPort, PodSecurityContext, + PodSpec, PodTemplateSpec, Probe, ResourceRequirements, TCPSocketAction, Volume, + VolumeMount, + }, + }, + kvp::{Annotation, Label, Labels}, + role_utils::RoleGroupRef, + utils::COMMON_BASH_TRAP_FUNCTIONS, +}; +use stackable_operator::k8s_openapi::{ + DeepMerge, + apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector, util::intstr::IntOrString}, +}; + +use crate::{ + controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + crd::{ + AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowExecutor, AirflowRole, CONFIG_PATH, Container, + HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, + METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, + TEMPLATE_VOLUME_NAME, + }, + framework::{ + self, + builder::meta::ownerreference_from_resource, + types::operator::*, + }, + product_logging::extend_config_map_with_log_config, + service::stateful_set_service_name, +}; + +use super::super::{ + AIRFLOW_CONTROLLER_NAME, PrecomputedPodData, ValidatedAirflowCluster, + ValidatedRoleGroupConfig, +}; + +pub(crate) struct RoleGroupBuilder<'a> { + cluster: &'a ValidatedAirflowCluster, + role_group_config: &'a ValidatedRoleGroupConfig, + rolegroup_ref: RoleGroupRef, + airflow_role: AirflowRole, + main_container: Container, + pod_data: &'a PrecomputedPodData, +} + +impl<'a> RoleGroupBuilder<'a> { + pub(crate) fn new( + cluster: &'a ValidatedAirflowCluster, + role_group_config: &'a ValidatedRoleGroupConfig, + rolegroup_ref: RoleGroupRef, + airflow_role: AirflowRole, + main_container: Container, + pod_data: &'a PrecomputedPodData, + ) -> Self { + Self { + cluster, + role_group_config, + rolegroup_ref, + airflow_role, + main_container, + pod_data, + } + } + + pub(crate) fn build_config_map(&self) -> ConfigMap { + let metadata = self + .common_metadata(self.rolegroup_ref.object_name()) + .build(); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder.metadata(metadata); + + cm_builder.add_data( + AIRFLOW_CONFIG_FILENAME, + self.role_group_config.config_file_content.clone(), + ); + + extend_config_map_with_log_config( + &self.rolegroup_ref, + &self.main_container, + &self.role_group_config.logging.airflow_container, + self.role_group_config.logging.vector_container.as_ref(), + &mut cm_builder, + &self.cluster.image, + ); + + cm_builder + .build() + .expect("ConfigMap should build because metadata is set") + } + + pub(crate) fn build_stateful_set(&self) -> StatefulSet { + let restarter_label = Label::try_from(("restarter.stackable.tech/enabled", "true")) + .expect("static label is always valid"); + + let metadata = self + .common_metadata(self.rolegroup_ref.object_name()) + .with_label(restarter_label) + .build(); + + let template = self.build_pod_template(); + + let match_labels = { + framework::kvp::label::role_group_selector( + self.cluster, + &ProductName::from_str_unsafe(APP_NAME), + &RoleName::from_str_unsafe(&self.rolegroup_ref.role), + &RoleGroupName::from_str_unsafe(&self.rolegroup_ref.role_group), + ) + }; + + let pod_management_policy = match self.airflow_role { + AirflowRole::Scheduler => "OrderedReady", + AirflowRole::Webserver + | AirflowRole::Worker + | AirflowRole::DagProcessor + | AirflowRole::Triggerer => "Parallel", + } + .to_string(); + + let spec = StatefulSetSpec { + pod_management_policy: Some(pod_management_policy), + replicas: self.pod_data.replicas.map(i32::from), + selector: LabelSelector { + match_labels: Some(match_labels.into()), + ..LabelSelector::default() + }, + service_name: stateful_set_service_name(&self.rolegroup_ref), + template, + volume_claim_templates: self + .pod_data + .listener_volume_claim_template + .clone() + .map(|pvc| vec![pvc]), + ..StatefulSetSpec::default() + }; + + StatefulSet { + metadata, + spec: Some(spec), + status: None, + } + } + + fn build_pod_template(&self) -> PodTemplateSpec { + let pod_metadata = ObjectMetaBuilder::new() + .with_labels(self.recommended_labels()) + .with_annotation( + Annotation::try_from(( + "kubectl.kubernetes.io/default-container", + format!("{}", self.main_container), + )) + .expect("static annotation is always valid"), + ) + .build(); + + let airflow_container = self.build_airflow_container(); + let metrics_container = self.build_metrics_container(); + + let mut containers = vec![airflow_container, metrics_container]; + containers.extend(self.pod_data.git_sync_containers.clone()); + if let Some(vector_container) = &self.pod_data.vector_container { + containers.push(vector_container.clone()); + } + + let init_containers = if self.pod_data.git_sync_init_containers.is_empty() { + None + } else { + Some(self.pod_data.git_sync_init_containers.clone()) + }; + + let volumes = self.build_volumes(); + + let termination_grace_period_seconds = self + .role_group_config + .graceful_shutdown_timeout + .as_secs() + .try_into() + .ok(); + + let mut pod_template = PodTemplateSpec { + metadata: Some(pod_metadata), + spec: Some(PodSpec { + affinity: { + let a = &self.role_group_config.affinity; + if a.pod_affinity.is_some() + || a.pod_anti_affinity.is_some() + || a.node_affinity.is_some() + { + Some(Affinity { + pod_affinity: a.pod_affinity.clone(), + pod_anti_affinity: a.pod_anti_affinity.clone(), + node_affinity: a.node_affinity.clone(), + }) + } else { + None + } + }, + containers, + init_containers, + service_account_name: Some(self.pod_data.service_account_name.clone()), + termination_grace_period_seconds, + security_context: Some(PodSecurityContext { + fs_group: Some(1000), + ..PodSecurityContext::default() + }), + image_pull_secrets: self.cluster.image.pull_secrets.clone(), + volumes: if volumes.is_empty() { + None + } else { + Some(volumes) + }, + ..PodSpec::default() + }), + }; + + pod_template.merge_from(self.pod_data.pod_overrides.clone()); + pod_template + } + + fn build_airflow_container(&self) -> K8sContainer { + let mut volume_mounts = vec![ + VolumeMount { + name: CONFIG_VOLUME_NAME.to_string(), + mount_path: CONFIG_PATH.to_string(), + ..VolumeMount::default() + }, + VolumeMount { + name: LOG_CONFIG_VOLUME_NAME.to_string(), + mount_path: LOG_CONFIG_DIR.to_string(), + ..VolumeMount::default() + }, + VolumeMount { + name: LOG_VOLUME_NAME.to_string(), + mount_path: STACKABLE_LOG_DIR.to_string(), + ..VolumeMount::default() + }, + ]; + + volume_mounts.extend(self.pod_data.extra_volume_mounts.clone()); + volume_mounts.extend(self.pod_data.auth_volume_mounts.clone()); + volume_mounts.extend(self.pod_data.git_sync_volume_mounts.clone()); + + if matches!( + self.pod_data.executor, + AirflowExecutor::KubernetesExecutors { .. } + ) { + volume_mounts.push(VolumeMount { + name: TEMPLATE_VOLUME_NAME.to_string(), + mount_path: TEMPLATE_LOCATION.to_string(), + ..VolumeMount::default() + }); + } + + if self.airflow_role.get_http_port().is_some() + && self.pod_data.listener_volume_claim_template.is_some() + { + volume_mounts.push(VolumeMount { + name: LISTENER_VOLUME_NAME.to_string(), + mount_path: LISTENER_VOLUME_DIR.to_string(), + ..VolumeMount::default() + }); + } + + let mut ports = Vec::new(); + if let Some(http_port) = self.airflow_role.get_http_port() { + ports.push(ContainerPort { + name: Some(HTTP_PORT_NAME.to_string()), + container_port: http_port.into(), + ..ContainerPort::default() + }); + } + + let (readiness_probe, liveness_probe) = + if let Some(http_port) = self.airflow_role.get_http_port() { + let probe = Probe { + tcp_socket: Some(TCPSocketAction { + port: IntOrString::Int(http_port.into()), + ..TCPSocketAction::default() + }), + initial_delay_seconds: Some(60), + period_seconds: Some(10), + failure_threshold: Some(6), + ..Probe::default() + }; + (Some(probe.clone()), Some(probe)) + } else { + (None, None) + }; + + K8sContainer { + name: self.main_container.to_string(), + image: Some(self.cluster.image.image.clone()), + image_pull_policy: Some(self.cluster.image.image_pull_policy.clone()), + command: Some(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]), + args: Some(vec![self.pod_data.airflow_commands.join("\n")]), + env: Some(self.pod_data.env_vars.clone()), + ports: if ports.is_empty() { None } else { Some(ports) }, + volume_mounts: Some(volume_mounts), + resources: Some(self.role_group_config.resources.clone().into()), + readiness_probe, + liveness_probe, + ..K8sContainer::default() + } + } + + fn build_metrics_container(&self) -> K8sContainer { + let args = [ + COMMON_BASH_TRAP_FUNCTIONS.to_string(), + "prepare_signal_handlers".to_string(), + "/stackable/statsd_exporter &".to_string(), + "wait_for_termination $!".to_string(), + ] + .join("\n"); + + K8sContainer { + name: "metrics".to_string(), + image: Some(self.cluster.image.image.clone()), + image_pull_policy: Some(self.cluster.image.image_pull_policy.clone()), + command: Some(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]), + args: Some(vec![args]), + ports: Some(vec![ContainerPort { + name: Some(METRICS_PORT_NAME.to_string()), + container_port: METRICS_PORT.into(), + ..ContainerPort::default() + }]), + resources: Some(ResourceRequirements { + requests: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity("100m".to_string())), + ("memory".to_string(), Quantity("64Mi".to_string())), + ])), + limits: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity("200m".to_string())), + ("memory".to_string(), Quantity("64Mi".to_string())), + ])), + ..ResourceRequirements::default() + }), + ..K8sContainer::default() + } + } + + fn build_volumes(&self) -> Vec { + // REVIEW: controller_commons::create_volumes is called with the new validated type + // (&ValidatedContainerLogConfigChoice) instead of the old Option<&ContainerLogConfig>. + // This is safe because the logging config has already been validated during the + // validate stage. + let mut volumes = controller_commons::create_volumes( + &self.rolegroup_ref.object_name(), + &self.role_group_config.logging.airflow_container, + ); + + volumes.extend(self.pod_data.extra_volumes.clone()); + volumes.extend(self.pod_data.auth_volumes.clone()); + volumes.extend(self.pod_data.git_sync_volumes.clone()); + + if let Some(template_cm_name) = &self.pod_data.executor_template_configmap_name { + volumes.push(Volume { + name: TEMPLATE_VOLUME_NAME.to_string(), + config_map: Some( + stackable_operator::k8s_openapi::api::core::v1::ConfigMapVolumeSource { + name: template_cm_name.clone(), + ..Default::default() + }, + ), + ..Volume::default() + }); + } + + volumes + } + + fn common_metadata(&self, resource_name: impl Into) -> ObjectMetaBuilder { + let mut builder = ObjectMetaBuilder::new(); + + builder + .name(resource_name) + .namespace(&self.cluster.namespace) + .ownerreference(ownerreference_from_resource(self.cluster, None, Some(true))) + .with_labels(self.recommended_labels()); + + builder + } + + fn recommended_labels(&self) -> Labels { + framework::kvp::label::recommended_labels( + self.cluster, + &ProductName::from_str_unsafe(APP_NAME), + &ProductVersion::from_str_unsafe( + &self.cluster.image.app_version_label_value.to_string(), + ), + &OperatorName::from_str_unsafe(OPERATOR_NAME), + &ControllerName::from_str_unsafe(AIRFLOW_CONTROLLER_NAME), + &RoleName::from_str_unsafe(&self.rolegroup_ref.role), + &RoleGroupName::from_str_unsafe(&self.rolegroup_ref.role_group), + ) + } +} diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs new file mode 100644 index 00000000..d0b77d35 --- /dev/null +++ b/rust/operator-binary/src/controller/dereference.rs @@ -0,0 +1,107 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::commons::{ + product_image_selection::ResolvedProductImage, random_secret_creation, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::crd::{ + authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, + internal_secret::{FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY}, + v1alpha2, +}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to resolve product image"))] + ResolveProductImage { + source: stackable_operator::commons::product_image_selection::Error, + }, + + #[snafu(display("failed to apply authentication configuration"))] + InvalidAuthenticationConfig { + source: crate::crd::authentication::Error, + }, + + #[snafu(display("invalid authorization config"))] + InvalidAuthorizationConfig { + source: stackable_operator::commons::opa::Error, + }, + + #[snafu(display("failed to create internal secret"))] + InvalidInternalSecret { + source: random_secret_creation::Error, + }, +} + +pub struct DereferencedObjects { + pub resolved_product_image: ResolvedProductImage, + pub authentication_config: AirflowClientAuthenticationDetailsResolved, + pub authorization_config: AirflowAuthorizationResolved, +} + +pub async fn dereference( + client: &stackable_operator::client::Client, + airflow: &v1alpha2::AirflowCluster, + image_base_name: &str, + image_repository: &str, + pkg_version: &str, +) -> std::result::Result { + let resolved_product_image = airflow + .spec + .image + .resolve(image_base_name, image_repository, pkg_version) + .context(ResolveProductImageSnafu)?; + + let authentication_config = AirflowClientAuthenticationDetailsResolved::from( + &airflow.spec.cluster_config.authentication, + client, + ) + .await + .context(InvalidAuthenticationConfigSnafu)?; + + let authorization_config = AirflowAuthorizationResolved::from_authorization_config( + client, + airflow, + &airflow.spec.cluster_config.authorization, + ) + .await + .context(InvalidAuthorizationConfigSnafu)?; + + random_secret_creation::create_random_secret_if_not_exists( + &airflow.shared_internal_secret_secret_name(), + INTERNAL_SECRET_SECRET_KEY, + 256, + airflow, + client, + ) + .await + .context(InvalidInternalSecretSnafu)?; + + random_secret_creation::create_random_secret_if_not_exists( + &airflow.shared_jwt_secret_secret_name(), + JWT_SECRET_SECRET_KEY, + 256, + airflow, + client, + ) + .await + .context(InvalidInternalSecretSnafu)?; + + random_secret_creation::create_random_secret_if_not_exists( + &airflow.shared_fernet_key_secret_name(), + FERNET_KEY_SECRET_KEY, + 32, + airflow, + client, + ) + .await + .context(InvalidInternalSecretSnafu)?; + + Ok(DereferencedObjects { + resolved_product_image, + authentication_config, + authorization_config, + }) +} diff --git a/rust/operator-binary/src/controller/update_status.rs b/rust/operator-binary/src/controller/update_status.rs new file mode 100644 index 00000000..5c354555 --- /dev/null +++ b/rust/operator-binary/src/controller/update_status.rs @@ -0,0 +1,47 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::status::condition::{ + compute_conditions, operations::ClusterOperationsConditionBuilder, + statefulset::StatefulSetConditionBuilder, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::crd::{AirflowClusterStatus, OPERATOR_NAME, v1alpha2}; + +use super::{Applied, KubernetesResources}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to update status"))] + PatchStatus { + source: stackable_operator::client::Error, + }, +} + +pub(crate) async fn update_status( + client: &stackable_operator::client::Client, + airflow: &v1alpha2::AirflowCluster, + applied_resources: KubernetesResources, +) -> std::result::Result<(), Error> { + let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + for stateful_set in applied_resources.stateful_sets { + ss_cond_builder.add(stateful_set); + } + + let cluster_operation_cond_builder = + ClusterOperationsConditionBuilder::new(&airflow.spec.cluster_operation); + + let status = AirflowClusterStatus { + conditions: compute_conditions( + airflow, + &[&ss_cond_builder, &cluster_operation_cond_builder], + ), + }; + + client + .apply_patch_status(OPERATOR_NAME, airflow, &status) + .await + .context(PatchStatusSnafu)?; + + Ok(()) +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs new file mode 100644 index 00000000..a3bcf623 --- /dev/null +++ b/rust/operator-binary/src/controller/validate.rs @@ -0,0 +1,1320 @@ +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + str::FromStr, +}; + +use product_config::{ProductConfigManager, types::PropertyNameKind}; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + configmap::ConfigMapBuilder, + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + }, + }, + commons::product_image_selection::ResolvedProductImage, + crd::git_sync, + database_connections::drivers::{ + celery::CeleryDatabaseConnectionDetails, sqlalchemy::SqlAlchemyDatabaseConnectionDetails, + }, + k8s_openapi::{ + DeepMerge, + api::core::v1::{ + ConfigMap, Container as K8sContainer, PodTemplateSpec, Volume, VolumeMount, + }, + }, + kube::{ + Resource, ResourceExt, + runtime::reflector::ObjectRef, + }, + kvp::{Label, Labels}, + product_config_utils::{ + env_vars_from, env_vars_from_rolegroup_config, transform_all_roles_to_config, + validate_all_roles_and_groups_config, + }, + product_logging::{self, framework::LoggingError, spec::Logging}, + role_utils::RoleGroupRef, +}; +use stackable_operator::builder::pod::volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}; +use stackable_operator::kvp::ObjectLabels; +use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; + +use crate::{ + controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + crd::{ + AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowConfig, AirflowExecutor, AirflowRole, + CONFIG_PATH, Container, ExecutorConfig, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, + OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_NAME, + authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, + }, + authorization::AirflowAuthorizationResolved, + v1alpha2, + }, + env_vars, + framework::{ + product_logging::framework::{ + VectorContainerLogConfig, + validate_logging_configuration_for_container, + }, + types::{ + kubernetes::{ConfigMapName, NamespaceName, Uid}, + operator::ClusterName, + }, + }, + product_logging::extend_config_map_with_log_config, +}; + +use super::{ + AIRFLOW_CONTROLLER_NAME, PrecomputedPodData, ValidatedAirflowCluster, ValidatedLogging, + ValidatedRoleConfig, ValidatedRoleGroupConfig, + dereference::DereferencedObjects, +}; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to validate cluster name"))] + InvalidClusterName { + source: crate::framework::macros::attributed_string_type::Error, + }, + + #[snafu(display("object has no associated namespace"))] + ObjectHasNoNamespace, + + #[snafu(display("failed to validate cluster namespace"))] + InvalidClusterNamespace { + source: crate::framework::macros::attributed_string_type::Error, + }, + + #[snafu(display("object has no UID"))] + ObjectHasNoUid, + + #[snafu(display("failed to validate cluster UID"))] + InvalidClusterUid { + source: crate::framework::macros::attributed_string_type::Error, + }, + + #[snafu(display("failed to validate logging configuration"))] + ValidateLoggingConfig { + source: crate::framework::product_logging::framework::Error, + }, + + #[snafu(display("vectorAggregatorConfigMapName must be set when vector agent is enabled"))] + MissingVectorAggregatorConfigMapName, + + #[snafu(display("failed to parse vector aggregator ConfigMap name"))] + ParseVectorAggregatorConfigMapName { + source: crate::framework::macros::attributed_string_type::Error, + }, + + #[snafu(display("graceful shutdown timeout is not configured"))] + MissingGracefulShutdownTimeout, + + #[snafu(display("failed to resolve and merge config for role and role group"))] + FailedToResolveConfig { source: crate::crd::Error }, + + #[snafu(display("failed to construct Airflow configuration"))] + ConstructConfig { source: crate::config::Error }, + + #[snafu(display("failed to write config file"))] + BuildConfigFile { + source: product_config::flask_app_config_writer::FlaskAppConfigWriterError, + }, + + #[snafu(display("Failed to transform configs"))] + ProductConfigTransform { + source: stackable_operator::product_config_utils::Error, + }, + + #[snafu(display("invalid product config"))] + InvalidProductConfig { + source: stackable_operator::product_config_utils::Error, + }, + + #[snafu(display("could not parse Airflow role [{role}]"))] + UnidentifiedAirflowRole { + source: strum::ParseError, + role: String, + }, + + #[snafu(display("object defines no airflow config role"))] + NoAirflowRole, + + #[snafu(display("failed to build environment variables"))] + BuildEnvVars { source: crate::env_vars::Error }, + + #[snafu(display("invalid git-sync specification"))] + InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { source: LoggingError }, + + #[snafu(display("failed to add LDAP volumes and volume mounts"))] + AddLdapVolumesAndVolumeMounts { + source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, + }, + + #[snafu(display("failed to add TLS volumes and volume mounts"))] + AddTlsVolumesAndVolumeMounts { + source: stackable_operator::commons::tls_verification::TlsClientDetailsError, + }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, + }, + + #[snafu(display("failed to build labels"))] + BuildLabels { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("invalid container name"))] + InvalidContainerName { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to add volume mount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to add volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to serialize pod template"))] + PodTemplateSerde { source: serde_yaml::Error }, + + #[snafu(display("failed to build pod template ConfigMap"))] + PodTemplateConfigMap { + source: stackable_operator::builder::configmap::Error, + }, + + #[snafu(display("failed to build object metadata"))] + ObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build graceful shutdown config"))] + GracefulShutdown { + source: stackable_operator::builder::pod::Error, + }, +} + +type ValidateResult = std::result::Result; + +fn validate_logging( + logging: &Logging, + main_container: Container, + vector_aggregator_config_map_name: Option<&str>, +) -> ValidateResult { + let airflow_container = validate_logging_configuration_for_container(logging, main_container) + .context(ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let aggregator_name = vector_aggregator_config_map_name + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + ConfigMapName::from_str(aggregator_name) + .context(ParseVectorAggregatorConfigMapNameSnafu)?; + let log_config = validate_logging_configuration_for_container(logging, Container::Vector) + .context(ValidateLoggingConfigSnafu)?; + Some(VectorContainerLogConfig { log_config }) + } else { + None + }; + + let git_sync_container_log_config = logging.for_container(&Container::GitSync).into_owned(); + + Ok(ValidatedLogging { + airflow_container, + vector_container, + git_sync_container_log_config, + }) +} + +pub(crate) fn validate_airflow_config( + config: &AirflowConfig, + vector_aggregator_config_map_name: Option<&str>, + config_file_content: String, +) -> ValidateResult { + let logging = validate_logging( + &config.logging, + Container::Airflow, + vector_aggregator_config_map_name, + )?; + + let graceful_shutdown_timeout = config + .graceful_shutdown_timeout + .context(MissingGracefulShutdownTimeoutSnafu)?; + + Ok(ValidatedRoleGroupConfig { + resources: config.resources.clone(), + logging, + affinity: config.affinity.clone(), + graceful_shutdown_timeout, + config_file_content, + }) +} + +fn validate_executor_config( + config: &ExecutorConfig, + vector_aggregator_config_map_name: Option<&str>, + config_file_content: String, +) -> ValidateResult { + let logging = validate_logging( + &config.logging, + Container::Base, + vector_aggregator_config_map_name, + )?; + + let graceful_shutdown_timeout = config + .graceful_shutdown_timeout + .context(MissingGracefulShutdownTimeoutSnafu)?; + + Ok(ValidatedRoleGroupConfig { + resources: config.resources.clone(), + logging, + affinity: config.affinity.clone(), + graceful_shutdown_timeout, + config_file_content, + }) +} + +/// Generates the `webserver_config.py` content for a role group. +/// +/// This function is called during the validate stage so that the build stage can +/// construct ConfigMaps infallibly. +fn generate_config_file_content( + authentication_config: &AirflowClientAuthenticationDetailsResolved, + authorization_config: &AirflowAuthorizationResolved, + product_version: &str, + rolegroup_config_overrides: &HashMap< + PropertyNameKind, + BTreeMap, + >, +) -> ValidateResult { + use std::io::Write; + + use product_config::flask_app_config_writer; + use stackable_operator::product_config_utils::{ + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, + }; + + use crate::{ + config::{self, PYTHON_IMPORTS}, + crd::{AIRFLOW_CONFIG_FILENAME, AirflowConfigOptions}, + }; + + let mut config = BTreeMap::new(); + config::add_airflow_config( + &mut config, + authentication_config, + authorization_config, + product_version, + ) + .context(ConstructConfigSnafu)?; + + let mut file_overrides = rolegroup_config_overrides + .get(&PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.to_string())) + .cloned() + .unwrap_or_default(); + + config.append(&mut file_overrides); + + let mut config_file = Vec::new(); + + if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).expect("writing to Vec is infallible"); + } + + let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + + flask_app_config_writer::write::( + &mut config_file, + config.iter(), + PYTHON_IMPORTS, + ) + .context(BuildConfigFileSnafu)?; + + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).expect("writing to Vec is infallible"); + } + + Ok(String::from_utf8(config_file).expect("flask_app_config_writer produces valid UTF-8")) +} + +/// Top-level validation: runs product_config, merges/validates per-rolegroup configs, +/// generates config file contents, and assembles a [`ValidatedAirflowCluster`]. +pub(crate) fn validate_cluster( + airflow: &v1alpha2::AirflowCluster, + dereferenced: &DereferencedObjects, + product_config_manager: &ProductConfigManager, +) -> ValidateResult { + let vector_aggregator_config_map_name = airflow + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref(); + + // --- product_config transform + validate --- + let mut roles = HashMap::new(); + for role in AirflowRole::iter() { + if let Some(resolved_role) = airflow.get_role(&role) { + roles.insert( + role.to_string(), + ( + vec![ + PropertyNameKind::Env, + PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.into()), + ], + resolved_role.clone(), + ), + ); + } + } + + let role_config = transform_all_roles_to_config(airflow, &roles); + let validated_role_config = validate_all_roles_and_groups_config( + &dereferenced.resolved_product_image.product_version, + &role_config.context(ProductConfigTransformSnafu)?, + product_config_manager, + false, + false, + ) + .context(InvalidProductConfigSnafu)?; + + // --- compute database connection details (infallible) --- + let templating_mechanism = + stackable_operator::database_connections::TemplatingMechanism::BashEnvSubstitution; + let metadata_database_connection_details = airflow + .spec + .cluster_config + .metadata_database + .sqlalchemy_connection_details_with_templating("METADATA", &templating_mechanism); + let celery_database_connection_details = if let ( + Some(celery_results_backend), + Some(celery_broker), + ) = ( + &airflow.spec.cluster_config.celery_results_backend, + &airflow.spec.cluster_config.celery_broker, + ) { + // The celery results backend and celery broker only work with configured celeryExecutors. + // Emit a warning if celery executors were not configured properly. + if !matches!( + &airflow.spec.executor, + AirflowExecutor::CeleryExecutors { .. } + ) { + tracing::warn!( + "No `spec.celeryExecutors` configured, but `spec.clusterConfig.celeryResultsBackend` and `spec.clusterConfig.celeryBroker` are provided. This only works in combination with a celery executor!" + ) + } + let celery_results_backend = celery_results_backend + .celery_connection_details_with_templating( + "CELERY_RESULT_BACKEND", + &templating_mechanism, + ); + let celery_broker = celery_broker + .celery_connection_details_with_templating("CELERY_BROKER", &templating_mechanism); + Some((celery_results_backend, celery_broker)) + } else { + None + }; + + // --- compute auth volumes/mounts (fallible) --- + let (auth_volumes, auth_volume_mounts) = + compute_auth_volumes_and_mounts(&dereferenced.authentication_config)?; + + // --- service account name (matches build_rbac_resources output: "{cluster}-serviceaccount") --- + let service_account_name = format!("{}-serviceaccount", airflow.name_any()); + + // --- per-role/rolegroup validation --- + let mut validated_role_groups = BTreeMap::new(); + let mut all_precomputed_pod_data = BTreeMap::new(); + + for (role_name, role_config) in validated_role_config.iter() { + let airflow_role = + AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { + role: role_name.to_string(), + })?; + + let mut validated_groups = BTreeMap::new(); + let mut pod_data_groups = BTreeMap::new(); + + for (rolegroup_name, rolegroup_config) in role_config.iter() { + let rolegroup_ref = RoleGroupRef { + cluster: ObjectRef::from_obj(airflow), + role: role_name.into(), + role_group: rolegroup_name.into(), + }; + + let merged_airflow_config = airflow + .merged_config(&airflow_role, &rolegroup_ref) + .context(FailedToResolveConfigSnafu)?; + + let config_file_content = generate_config_file_content( + &dereferenced.authentication_config, + &dereferenced.authorization_config, + &dereferenced.resolved_product_image.product_version, + rolegroup_config, + )?; + + let validated_config = validate_airflow_config( + &merged_airflow_config, + vector_aggregator_config_map_name, + config_file_content, + )?; + + let pod_data = compute_precomputed_pod_data( + airflow, + &airflow_role, + &rolegroup_ref, + rolegroup_config, + &dereferenced.resolved_product_image, + &dereferenced.authentication_config, + &dereferenced.authorization_config, + &metadata_database_connection_details, + &celery_database_connection_details, + &validated_config.logging, + &auth_volumes, + &auth_volume_mounts, + &service_account_name, + )?; + + validated_groups.insert(rolegroup_name.clone(), validated_config); + pod_data_groups.insert(rolegroup_name.clone(), pod_data); + } + + validated_role_groups.insert(airflow_role.clone(), validated_groups); + all_precomputed_pod_data.insert(airflow_role, pod_data_groups); + } + + // --- per-role config (PDB, listeners) --- + let mut validated_role_configs_map = BTreeMap::new(); + for role in AirflowRole::iter() { + if let Some(role_config) = airflow.role_config(&role) { + let pdb = &role_config.pod_disruption_budget; + let listener_class = role.listener_class_name(airflow); + let group_listener_name = airflow.group_listener_name(&role); + validated_role_configs_map.insert( + role, + ValidatedRoleConfig { + pdb_enabled: pdb.enabled, + pdb_max_unavailable: pdb.max_unavailable, + listener_class, + group_listener_name, + }, + ); + } + } + + // --- executor template config maps --- + let executor_template_config_maps = if let AirflowExecutor::KubernetesExecutors { + common_configuration, + } = &airflow.spec.executor + { + let merged_executor_config = airflow + .merged_executor_config(&common_configuration.config) + .context(FailedToResolveConfigSnafu)?; + + let config_file_content = generate_config_file_content( + &dereferenced.authentication_config, + &dereferenced.authorization_config, + &dereferenced.resolved_product_image.product_version, + &HashMap::new(), + )?; + + let validated_config = validate_executor_config( + &merged_executor_config, + vector_aggregator_config_map_name, + config_file_content, + )?; + + build_executor_template_config_maps( + airflow, + &dereferenced.resolved_product_image, + &dereferenced.authentication_config, + &metadata_database_connection_details, + &service_account_name, + &validated_config, + common_configuration, + )? + } else { + Vec::new() + }; + + // --- assemble --- + validate( + airflow, + &dereferenced.resolved_product_image, + validated_role_groups, + all_precomputed_pod_data, + executor_template_config_maps, + validated_role_configs_map, + ) +} + +/// Validates the AirflowCluster and produces a [`ValidatedAirflowCluster`] containing +/// all role groups with their validated configs. +fn validate( + airflow: &v1alpha2::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + validated_role_configs: BTreeMap>, + precomputed_pod_data: BTreeMap>, + executor_template_config_maps: Vec, + role_configs: BTreeMap, +) -> ValidateResult { + let cluster_name = + ClusterName::from_str(&airflow.name_any()).context(InvalidClusterNameSnafu)?; + let namespace = NamespaceName::from_str( + &airflow + .namespace() + .context(ObjectHasNoNamespaceSnafu)?, + ) + .context(InvalidClusterNamespaceSnafu)?; + let uid = Uid::from_str( + airflow + .meta() + .uid + .as_deref() + .context(ObjectHasNoUidSnafu)?, + ) + .context(InvalidClusterUidSnafu)?; + + Ok(ValidatedAirflowCluster::new( + resolved_product_image.clone(), + cluster_name, + namespace, + uid, + validated_role_configs, + precomputed_pod_data, + executor_template_config_maps, + role_configs, + airflow.spec.executor.clone(), + )) +} + +/// Extracts auth volumes and volume mounts using temporary builders. +/// +/// The upstream LDAP/TLS provider APIs require `PodBuilder`/`ContainerBuilder` references. +/// We create temporary builders, call the auth methods, then extract the raw volumes and mounts. +fn compute_auth_volumes_and_mounts( + authentication_config: &AirflowClientAuthenticationDetailsResolved, +) -> ValidateResult<(Vec, Vec)> { + let mut pb = PodBuilder::new(); + let mut cb = ContainerBuilder::new("dummy").expect("'dummy' is a valid container name"); + + let mut ldap_providers = BTreeSet::new(); + let mut tls_credentials = BTreeSet::new(); + + for auth_class in &authentication_config.authentication_classes_resolved { + match auth_class { + AirflowAuthenticationClassResolved::Ldap { provider } => { + ldap_providers.insert(provider); + } + AirflowAuthenticationClassResolved::Oidc { provider, .. } => { + tls_credentials.insert(&provider.tls); + } + } + } + + for provider in ldap_providers { + provider + .add_volumes_and_mounts(&mut pb, vec![&mut cb]) + .context(AddLdapVolumesAndVolumeMountsSnafu)?; + } + for tls in tls_credentials { + tls.add_volumes_and_mounts(&mut pb, vec![&mut cb]) + .context(AddTlsVolumesAndVolumeMountsSnafu)?; + } + + let container = cb.build(); + let pod_template = pb.build_template(); + + let volumes = pod_template + .spec + .and_then(|s| s.volumes) + .unwrap_or_default(); + let mounts = container.volume_mounts.unwrap_or_default(); + + Ok((volumes, mounts)) +} + +/// Builds the executor template ConfigMaps for KubernetesExecutor mode. +/// +/// Produces two ConfigMaps: +/// 1. A logging/config ConfigMap for the executor pods (equivalent to a rolegroup ConfigMap) +/// 2. A pod template ConfigMap containing a serialised PodTemplate that Airflow uses to +/// launch executor pods +/// +/// This is done in the validate stage because it uses PodBuilder/ContainerBuilder which +/// are fallible. The build stage then just passes these through to KubernetesResources. +#[allow(clippy::too_many_arguments)] +fn build_executor_template_config_maps( + airflow: &v1alpha2::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + authentication_config: &AirflowClientAuthenticationDetailsResolved, + metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, + service_account_name: &str, + validated_config: &ValidatedRoleGroupConfig, + common_configuration: &crate::crd::AirflowExecutorCommonConfiguration, +) -> ValidateResult> { + let executor_rolegroup_ref = RoleGroupRef { + cluster: ObjectRef::from_obj(airflow), + role: "executor".into(), + role_group: "kubernetes".into(), + }; + + // 1. Build the executor logging/config ConfigMap + let executor_config_cm = { + let metadata = ObjectMetaBuilder::new() + .name(executor_rolegroup_ref.object_name()) + .namespace_opt(airflow.namespace()) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(ObjectMetaSnafu)? + .with_recommended_labels(&build_object_labels( + airflow, + resolved_product_image, + "executor", + "executor-template", + )) + .context(ObjectMetaSnafu)? + .build(); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder.metadata(metadata); + cm_builder.add_data( + AIRFLOW_CONFIG_FILENAME, + validated_config.config_file_content.clone(), + ); + + extend_config_map_with_log_config( + &executor_rolegroup_ref, + &Container::Base, + &validated_config.logging.airflow_container, + validated_config.logging.vector_container.as_ref(), + &mut cm_builder, + resolved_product_image, + ); + + cm_builder + .build() + .context(PodTemplateConfigMapSnafu)? + }; + + // 2. Build the executor pod template ConfigMap + let executor_template_cm = { + // git-sync resources for the executor template + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( + &airflow.spec.cluster_config.dags_git_sync, + resolved_product_image, + &env_vars_from(&common_configuration.env_overrides), + &airflow.volume_mounts(), + LOG_VOLUME_NAME, + &validated_config.logging.git_sync_container_log_config, + ) + .context(InvalidGitSyncSpecSnafu)?; + + let mut pb = PodBuilder::new(); + let pb_metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&build_object_labels( + airflow, + resolved_product_image, + "executor", + "executor-template", + )) + .context(ObjectMetaSnafu)? + .build(); + + pb.metadata(pb_metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .affinity(&validated_config.affinity) + .service_account_name(service_account_name) + .restart_policy("Never") + .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + + pb.termination_grace_period(&validated_config.graceful_shutdown_timeout) + .context(GracefulShutdownSnafu)?; + + // Container name "base" is an Airflow requirement + let mut airflow_container = ContainerBuilder::new(&Container::Base.to_string()) + .context(InvalidContainerNameSnafu)?; + + // Auth volumes and mounts + add_authentication_volumes_and_volume_mounts_to_builders( + authentication_config, + &mut airflow_container, + &mut pb, + )?; + + airflow_container + .image_from_product_image(resolved_product_image) + .resources(validated_config.resources.clone().into()) + .add_env_vars(env_vars::build_airflow_template_envs( + airflow, + &common_configuration.env_overrides, + validated_config.logging.is_vector_agent_enabled(), + metadata_database_connection_details, + &git_sync_resources, + resolved_product_image, + )) + .add_volume_mounts(airflow.volume_mounts()) + .context(AddVolumeMountSnafu)? + .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)?; + + // Git-sync resources (init containers only, no sidecars for executor template) + for container in git_sync_resources.git_sync_init_containers.iter().cloned() { + pb.add_init_container(container); + } + pb.add_volumes(git_sync_resources.git_content_volumes.clone()) + .context(AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ssh_volumes.clone()) + .context(AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ca_cert_volumes.clone()) + .context(AddVolumeSnafu)?; + airflow_container + .add_volume_mounts(git_sync_resources.git_content_volume_mounts.clone()) + .context(AddVolumeMountSnafu)?; + + // Database connection env vars + metadata_database_connection_details.add_to_container(&mut airflow_container); + + pb.add_container(airflow_container.build()); + pb.add_volumes(airflow.volumes().clone()) + .context(AddVolumeSnafu)?; + // REVIEW: controller_commons::create_volumes now takes &ValidatedContainerLogConfigChoice + // instead of Option<&ContainerLogConfig>. The validated type ensures logging config + // has already been checked, so the build stage can use it directly. + pb.add_volumes(controller_commons::create_volumes( + &executor_rolegroup_ref.object_name(), + &validated_config.logging.airflow_container, + )) + .context(AddVolumeSnafu)?; + + if let Some(vector_config) = &validated_config.logging.vector_container { + let vector_aggregator_config_map_name = airflow + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + pb.add_container(build_logging_container( + resolved_product_image, + vector_config, + vector_aggregator_config_map_name, + )?); + } + + let mut pod_template = pb.build_template(); + pod_template.merge_from(common_configuration.pod_overrides.clone()); + + let restarter_label = Label::try_from(("restarter.stackable.tech/enabled", "true")) + .expect("static label is always valid"); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(airflow) + .name(airflow.executor_template_configmap_name()) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(ObjectMetaSnafu)? + .with_recommended_labels(&build_object_labels( + airflow, + resolved_product_image, + "executor", + "executor-template", + )) + .context(ObjectMetaSnafu)? + .with_label(restarter_label) + .build(), + ) + .add_data( + TEMPLATE_NAME, + serde_yaml::to_string(&pod_template) + .context(PodTemplateSerdeSnafu)?, + ); + + cm_builder + .build() + .context(PodTemplateConfigMapSnafu)? + }; + + Ok(vec![executor_config_cm, executor_template_cm]) +} + +/// Helper to add authentication volumes and volume mounts directly to builders. +/// Used by the executor template where we build a PodTemplate using PodBuilder/ContainerBuilder. +fn add_authentication_volumes_and_volume_mounts_to_builders( + authentication_config: &AirflowClientAuthenticationDetailsResolved, + cb: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> ValidateResult<()> { + let mut ldap_providers = BTreeSet::new(); + let mut tls_credentials = BTreeSet::new(); + + for auth_class in &authentication_config.authentication_classes_resolved { + match auth_class { + AirflowAuthenticationClassResolved::Ldap { provider } => { + ldap_providers.insert(provider); + } + AirflowAuthenticationClassResolved::Oidc { provider, .. } => { + tls_credentials.insert(&provider.tls); + } + } + } + + for provider in ldap_providers { + provider + .add_volumes_and_mounts(pb, vec![cb]) + .context(AddLdapVolumesAndVolumeMountsSnafu)?; + } + for tls in tls_credentials { + tls.add_volumes_and_mounts(pb, vec![cb]) + .context(AddTlsVolumesAndVolumeMountsSnafu)?; + } + Ok(()) +} + +fn build_object_labels<'a>( + airflow: &'a v1alpha2::AirflowCluster, + resolved_product_image: &'a ResolvedProductImage, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, v1alpha2::AirflowCluster> { + ObjectLabels { + owner: airflow, + app_name: APP_NAME, + app_version: &resolved_product_image.app_version_label_value, + operator_name: OPERATOR_NAME, + controller_name: AIRFLOW_CONTROLLER_NAME, + role, + role_group, + } +} + +fn build_logging_container( + resolved_product_image: &ResolvedProductImage, + vector_config: &VectorContainerLogConfig, + vector_aggregator_config_map_name: &str, +) -> ValidateResult { + let raw_log_config = vector_config.log_config.to_raw_container_log_config(); + + product_logging::framework::vector_container( + resolved_product_image, + CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, + Some(&raw_log_config), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu) +} + +/// Computes all pod-level data needed by the build stage to construct StatefulSets infallibly. +#[allow(clippy::too_many_arguments)] +fn compute_precomputed_pod_data( + airflow: &v1alpha2::AirflowCluster, + airflow_role: &AirflowRole, + rolegroup_ref: &RoleGroupRef, + rolegroup_config: &HashMap>, + resolved_product_image: &ResolvedProductImage, + authentication_config: &AirflowClientAuthenticationDetailsResolved, + authorization_config: &AirflowAuthorizationResolved, + metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, + celery_database_connection_details: &Option<( + CeleryDatabaseConnectionDetails, + CeleryDatabaseConnectionDetails, + )>, + validated_logging: &ValidatedLogging, + auth_volumes: &[Volume], + auth_volume_mounts: &[VolumeMount], + service_account_name: &str, +) -> ValidateResult { + let executor = &airflow.spec.executor; + + // --- git-sync resources --- + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( + &airflow.spec.cluster_config.dags_git_sync, + resolved_product_image, + &env_vars_from_rolegroup_config(rolegroup_config), + &airflow.volume_mounts(), + LOG_VOLUME_NAME, + &validated_logging.git_sync_container_log_config, + ) + .context(InvalidGitSyncSpecSnafu)?; + + // --- env vars --- + let mut env_vars = env_vars::build_airflow_statefulset_envs( + airflow, + airflow_role, + rolegroup_config, + executor, + authentication_config, + authorization_config, + metadata_database_connection_details, + celery_database_connection_details, + &git_sync_resources, + resolved_product_image, + ) + .context(BuildEnvVarsSnafu)?; + + // Database connection details add secret-referenced env vars via ContainerBuilder. + // Extract them using a temp builder. + let db_env_vars = { + let mut cb = ContainerBuilder::new("dummy").expect("'dummy' is a valid container name"); + metadata_database_connection_details.add_to_container(&mut cb); + if let Some((celery_result_backend, celery_broker)) = celery_database_connection_details { + celery_result_backend.add_to_container(&mut cb); + celery_broker.add_to_container(&mut cb); + } + cb.build().env.unwrap_or_default() + }; + env_vars.extend(db_env_vars); + + // --- commands --- + let airflow_commands = + airflow_role.get_commands(airflow, authentication_config, resolved_product_image); + + // --- git-sync containers/volumes --- + let use_git_sync_init_containers = matches!(executor, AirflowExecutor::CeleryExecutors { .. }); + let git_sync_containers = git_sync_resources.git_sync_containers.clone(); + let git_sync_init_containers = if use_git_sync_init_containers { + git_sync_resources.git_sync_init_containers.clone() + } else { + Vec::new() + }; + let mut git_sync_volumes = git_sync_resources.git_content_volumes.clone(); + git_sync_volumes.extend(git_sync_resources.git_ssh_volumes.clone()); + git_sync_volumes.extend(git_sync_resources.git_ca_cert_volumes.clone()); + let git_sync_volume_mounts = git_sync_resources.git_content_volume_mounts.clone(); + + // --- vector container --- + let vector_container = if let Some(vector_config) = &validated_logging.vector_container { + let vector_aggregator_config_map_name = airflow + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + Some(build_logging_container( + resolved_product_image, + vector_config, + vector_aggregator_config_map_name, + )?) + } else { + None + }; + + // --- replicas --- + let binding = airflow.get_role(airflow_role); + let role = binding.as_ref().context(NoAirflowRoleSnafu)?; + let rolegroup = role.role_groups.get(&rolegroup_ref.role_group); + let replicas = rolegroup.and_then(|rg| rg.replicas); + + // --- pod overrides --- + let mut pod_overrides = PodTemplateSpec::default(); + pod_overrides.merge_from(role.config.pod_overrides.clone()); + if let Some(rg) = rolegroup { + pod_overrides.merge_from(rg.config.pod_overrides.clone()); + } + + // --- executor template configmap name --- + let executor_template_configmap_name = + if matches!(executor, AirflowExecutor::KubernetesExecutors { .. }) { + Some(airflow.executor_template_configmap_name()) + } else { + None + }; + + // --- listener PVC --- + let listener_volume_claim_template = if airflow_role.get_http_port().is_some() { + if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) { + let unversioned_labels = Labels::recommended(&ObjectLabels { + owner: airflow, + app_name: APP_NAME, + app_version: "none", + operator_name: OPERATOR_NAME, + controller_name: AIRFLOW_CONTROLLER_NAME, + role: &rolegroup_ref.role, + role_group: &rolegroup_ref.role_group, + }) + .context(BuildLabelsSnafu)?; + + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(listener_group_name), + &unversioned_labels, + ) + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + Some(pvc) + } else { + None + } + } else { + None + }; + + // --- user-defined extra volumes/mounts from CRD --- + let extra_volumes = airflow.volumes().clone(); + let extra_volume_mounts = airflow.volume_mounts(); + + Ok(PrecomputedPodData { + env_vars, + airflow_commands, + auth_volumes: auth_volumes.to_vec(), + auth_volume_mounts: auth_volume_mounts.to_vec(), + extra_volumes, + extra_volume_mounts, + git_sync_containers, + git_sync_init_containers, + git_sync_volumes, + git_sync_volume_mounts, + vector_container, + service_account_name: service_account_name.to_string(), + replicas, + pod_overrides, + executor: executor.clone(), + executor_template_configmap_name, + listener_volume_claim_template, + }) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + kube::api::ObjectMeta, + kvp::LabelValue, + product_logging::spec::{ + AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging, + }, + shared::time::Duration, + }; + + use super::*; + use crate::crd::{AirflowConfig, Container}; + + // --- validate tests --- + + fn airflow_config_with_logging(enable_vector: bool) -> AirflowConfig { + let mut containers = BTreeMap::new(); + containers.insert( + Container::Airflow, + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + ); + if enable_vector { + containers.insert( + Container::Vector, + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + ); + } + AirflowConfig { + resources: Default::default(), + logging: Logging { + enable_vector_agent: enable_vector, + containers, + }, + affinity: Default::default(), + graceful_shutdown_timeout: Some(Duration::from_secs(120)), + } + } + + #[test] + fn test_validate_airflow_config_without_vector() { + let config = airflow_config_with_logging(false); + let result = validate_airflow_config(&config, None, String::new()); + assert!(result.is_ok()); + let validated = result.unwrap(); + assert!(validated.logging.vector_container.is_none()); + assert!(!validated.logging.is_vector_agent_enabled()); + assert_eq!( + validated.graceful_shutdown_timeout, + Duration::from_secs(120) + ); + } + + #[test] + fn test_validate_airflow_config_with_vector() { + let config = airflow_config_with_logging(true); + let result = + validate_airflow_config(&config, Some("vector-aggregator-discovery"), String::new()); + assert!(result.is_ok()); + let validated = result.unwrap(); + assert!(validated.logging.vector_container.is_some()); + assert!(validated.logging.is_vector_agent_enabled()); + } + + #[test] + fn test_validate_vector_enabled_missing_config_map() { + let config = airflow_config_with_logging(true); + let result = validate_airflow_config(&config, None, String::new()); + assert!(result.is_err()); + } + + #[test] + fn test_validate_missing_graceful_shutdown() { + let mut config = airflow_config_with_logging(false); + config.graceful_shutdown_timeout = None; + let result = validate_airflow_config(&config, None, String::new()); + assert!(result.is_err()); + } + + #[test] + fn test_validate_ok() { + let (airflow, image) = test_objects(); + let result = validate( + &airflow, + &image, + BTreeMap::new(), + BTreeMap::new(), + vec![], + BTreeMap::new(), + ); + assert!(result.is_ok()); + let validated = result.unwrap(); + assert_eq!(validated.name.to_string(), "my-airflow"); + assert_eq!(validated.namespace.to_string(), "default"); + } + + #[test] + fn test_validate_err_missing_name() { + test_validate_err( + |airflow, _| airflow.metadata.name = None, + ErrorDiscriminants::InvalidClusterName, + ); + } + + #[test] + fn test_validate_err_missing_namespace() { + test_validate_err( + |airflow, _| airflow.metadata.namespace = None, + ErrorDiscriminants::ObjectHasNoNamespace, + ); + } + + #[test] + fn test_validate_err_missing_uid() { + test_validate_err( + |airflow, _| airflow.metadata.uid = None, + ErrorDiscriminants::ObjectHasNoUid, + ); + } + + #[test] + fn test_validate_err_invalid_cluster_name() { + test_validate_err( + |airflow, _| { + airflow.metadata.name = + Some("THIS-IS-NOT-A-VALID-DNS-LABEL-NAME-BECAUSE-UPPERCASE".to_string()) + }, + ErrorDiscriminants::InvalidClusterName, + ); + } + + #[test] + fn test_validate_err_invalid_namespace() { + test_validate_err( + |airflow, _| airflow.metadata.namespace = Some("INVALID NAMESPACE".to_string()), + ErrorDiscriminants::InvalidClusterNamespace, + ); + } + + #[test] + fn test_validate_err_invalid_uid() { + test_validate_err( + |airflow, _| airflow.metadata.uid = Some("not-a-uuid".to_string()), + ErrorDiscriminants::InvalidClusterUid, + ); + } + + fn test_validate_err( + mutate: fn(&mut v1alpha2::AirflowCluster, &mut ResolvedProductImage), + expected: ErrorDiscriminants, + ) { + let (mut airflow, mut image) = test_objects(); + mutate(&mut airflow, &mut image); + let result = validate( + &airflow, + &image, + BTreeMap::new(), + BTreeMap::new(), + vec![], + BTreeMap::new(), + ); + match result { + Err(err) => assert_eq!(expected, ErrorDiscriminants::from(err)), + Ok(_) => panic!("validate should have failed with {expected:?}"), + } + } + + fn test_objects() -> (v1alpha2::AirflowCluster, ResolvedProductImage) { + let airflow = v1alpha2::AirflowCluster { + metadata: ObjectMeta { + name: Some("my-airflow".to_string()), + namespace: Some("default".to_string()), + uid: Some("e6ac237d-a6d4-43a1-8135-f36506110912".to_string()), + ..ObjectMeta::default() + }, + spec: serde_json::from_value(serde_json::json!({ + "image": { "productVersion": "2.10.4" }, + "clusterConfig": { + "credentialsSecretName": "airflow-admin-credentials", + "metadataDatabase": { + "postgresql": { + "host": "airflow-postgresql", + "database": "airflow", + "credentialsSecretName": "airflow-postgresql-credentials" + } + } + }, + "kubernetesExecutors": { "config": {} }, + "webservers": { "roleGroups": { "default": { "config": {} } } }, + "schedulers": { "roleGroups": { "default": { "config": {} } } } + })) + .expect("test spec JSON should be valid"), + status: None, + }; + + let image = ResolvedProductImage { + product_version: "2.10.4".to_owned(), + app_version_label_value: LabelValue::from_str("2.10.4-stackable0.0.0-dev") + .expect("valid label value"), + image: "oci.stackable.tech/sdp/airflow:2.10.4-stackable0.0.0-dev".to_string(), + image_pull_policy: "Always".to_owned(), + pull_secrets: None, + }; + + (airflow, image) + } +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 8bc2b586..85ffe32c 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -33,12 +33,12 @@ use stackable_operator::{ }; use crate::{ - airflow_controller::AIRFLOW_FULL_CONTROLLER_NAME, + controller::AIRFLOW_FULL_CONTROLLER_NAME, crd::{AirflowCluster, AirflowClusterVersion, OPERATOR_NAME, v1alpha1, v1alpha2}, webhooks::conversion::create_webhook_server, }; -mod airflow_controller; +mod controller; mod config; mod controller_commons; mod crd; @@ -178,9 +178,9 @@ async fn main() -> anyhow::Result<()> { .graceful_shutdown_on(sigterm_watcher.handle()) .run( // REVIEW: renamed from reconcile_airflow to reconcile - airflow_controller::reconcile, - airflow_controller::error_policy, - Arc::new(airflow_controller::Ctx { + controller::reconcile, + controller::error_policy, + Arc::new(controller::Ctx { client: client.clone(), operator_environment, product_config, From 0f52c40dfa225cbe43d6da680a916723b2098a47 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 11:26:43 +0200 Subject: [PATCH 13/14] refactor: shorten test role group names to fit 16-char limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RoleGroupName has a max_length of 16 characters. Rename test role groups to fit within this limit: - resources-from-role (19) → from-role (9) - resources-from-role-group (25) → from-rg (7) - resources-from-pod-overrides (28) → from-overrides (14) - automatic-log-config (20) → auto-log-cfg (12) - custom-log-config (17) → custom-log-cfg (14) Co-Authored-By: Claude Opus 4.6 --- .../templates/kuttl/logging/41-assert.yaml.j2 | 12 +++---- .../41-install-airflow-cluster.yaml.j2 | 16 +++++----- .../templates/kuttl/logging/52-assert.yaml.j2 | 8 ++--- .../templates/kuttl/logging/70-assert.yaml.j2 | 8 ++--- .../airflow-vector-aggregator-values.yaml.j2 | 32 +++++++++---------- .../kuttl/resources/30-assert.yaml.j2 | 6 ++-- .../30-install-airflow-cluster.yaml.j2 | 6 ++-- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/templates/kuttl/logging/41-assert.yaml.j2 b/tests/templates/kuttl/logging/41-assert.yaml.j2 index 77370e89..9f150840 100644 --- a/tests/templates/kuttl/logging/41-assert.yaml.j2 +++ b/tests/templates/kuttl/logging/41-assert.yaml.j2 @@ -8,7 +8,7 @@ timeout: 1200 apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-webserver-automatic-log-config + name: airflow-webserver-auto-log-cfg status: readyReplicas: 1 replicas: 1 @@ -16,7 +16,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-webserver-custom-log-config + name: airflow-webserver-custom-log-cfg status: readyReplicas: 1 replicas: 1 @@ -25,7 +25,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-worker-automatic-log-config + name: airflow-worker-auto-log-cfg status: readyReplicas: 1 replicas: 1 @@ -33,7 +33,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-worker-custom-log-config + name: airflow-worker-custom-log-cfg status: readyReplicas: 1 replicas: 1 @@ -42,7 +42,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-scheduler-automatic-log-config + name: airflow-scheduler-auto-log-cfg status: readyReplicas: 1 replicas: 1 @@ -50,7 +50,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-scheduler-custom-log-config + name: airflow-scheduler-custom-log-cfg status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index d93a44da..4c079cc2 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -121,7 +121,7 @@ spec: memory: limit: 3Gi roleGroups: - automatic-log-config: + auto-log-cfg: replicas: 1 config: logging: @@ -162,7 +162,7 @@ spec: - name: prepared-logs configMap: name: prepared-logs - custom-log-config: + custom-log-cfg: replicas: 1 config: logging: @@ -181,7 +181,7 @@ spec: memory: limit: 3Gi roleGroups: - automatic-log-config: + auto-log-cfg: replicas: 1 config: logging: @@ -215,7 +215,7 @@ spec: loggers: ROOT: level: INFO - custom-log-config: + custom-log-cfg: replicas: 1 config: logging: @@ -233,7 +233,7 @@ spec: max: 250m memory: limit: 512Mi - # automatic-log-config + # auto-log-cfg logging: enableVectorAgent: true containers: @@ -261,7 +261,7 @@ spec: loggers: ROOT: level: INFO - # custom-log-config is not tested for kubernetesExecutors because + # custom-log-cfg is not tested for kubernetesExecutors because # there are no roleGroups to test both {% endif %} schedulers: @@ -273,7 +273,7 @@ spec: memory: limit: 1Gi roleGroups: - automatic-log-config: + auto-log-cfg: replicas: 1 config: logging: @@ -303,7 +303,7 @@ spec: loggers: ROOT: level: INFO - custom-log-config: + custom-log-cfg: replicas: 1 config: logging: diff --git a/tests/templates/kuttl/logging/52-assert.yaml.j2 b/tests/templates/kuttl/logging/52-assert.yaml.j2 index 35442128..83c0b6fe 100644 --- a/tests/templates/kuttl/logging/52-assert.yaml.j2 +++ b/tests/templates/kuttl/logging/52-assert.yaml.j2 @@ -7,10 +7,10 @@ timeout: 600 commands: {% if test_scenario['values']['airflow'].find(",") > 0 %} - script: | - kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group automatic-log-config --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" - kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group custom-log-config --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" + kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group auto-log-cfg --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" + kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group custom-log-cfg --airflow-version "{{ test_scenario['values']['airflow'].split(',')[0] }}" {% else %} - script: | - kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group automatic-log-config --airflow-version "{{ test_scenario['values']['airflow'] }}" - kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group custom-log-config --airflow-version "{{ test_scenario['values']['airflow'] }}" + kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group auto-log-cfg --airflow-version "{{ test_scenario['values']['airflow'] }}" + kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/metrics.py --role-group custom-log-cfg --airflow-version "{{ test_scenario['values']['airflow'] }}" {% endif %} diff --git a/tests/templates/kuttl/logging/70-assert.yaml.j2 b/tests/templates/kuttl/logging/70-assert.yaml.j2 index 36ff53a9..faa1adc0 100644 --- a/tests/templates/kuttl/logging/70-assert.yaml.j2 +++ b/tests/templates/kuttl/logging/70-assert.yaml.j2 @@ -19,17 +19,17 @@ commands: # has to be declared. # See https://github.com/apache/airflow/pull/52581. - # Rolegroup custom-log-config + # Rolegroup custom-log-cfg CURL_RESPONSE_CUSTOM=$( - kubectl -n $NAMESPACE exec airflow-webserver-custom-log-config-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-custom-log-config-headless:8793/log 2>/dev/null || true);echo "$CODE"' + kubectl -n $NAMESPACE exec airflow-webserver-custom-log-cfg-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-custom-log-cfg-headless:8793/log 2>/dev/null || true);echo "$CODE"' ) # Log-Endpoint Test Assertions: echo "The HTTP Code is $CURL_RESPONSE_CUSTOM (an internal JWT token is needed for full access)" - # Rolegroup automatic-log-config + # Rolegroup auto-log-cfg CURL_RESPONSE_AUTO=$( - kubectl -n $NAMESPACE exec airflow-webserver-automatic-log-config-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-automatic-log-config-headless:8793/log 2>/dev/null || true);echo "$CODE"' + kubectl -n $NAMESPACE exec airflow-webserver-auto-log-cfg-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-auto-log-cfg-headless:8793/log 2>/dev/null || true);echo "$CODE"' ) echo "The HTTP Code is $CURL_RESPONSE_AUTO (an internal JWT token is needed for full access)" [ "$CURL_RESPONSE_CUSTOM" -eq 403 ] && [ "$CURL_RESPONSE_AUTO" -eq 403 ] diff --git a/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 index 714e3476..bf1a76c1 100644 --- a/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/airflow-vector-aggregator-values.yaml.j2 @@ -28,98 +28,98 @@ customConfig: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-webserver-automatic-log-config-0" && + .pod == "airflow-webserver-auto-log-cfg-0" && .container == "airflow" filteredAutomaticLogConfigWebserverGitSync: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-webserver-automatic-log-config-0" && + .pod == "airflow-webserver-auto-log-cfg-0" && .container == "git-sync-0" filteredAutomaticLogConfigWebserverVector: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-webserver-automatic-log-config-0" && + .pod == "airflow-webserver-auto-log-cfg-0" && .container == "vector" filteredCustomLogConfigWebserverAirflow: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-webserver-custom-log-config-0" && + .pod == "airflow-webserver-custom-log-cfg-0" && .container == "airflow" filteredCustomLogConfigWebserverVector: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-webserver-custom-log-config-0" && + .pod == "airflow-webserver-custom-log-cfg-0" && .container == "vector" filteredAutomaticLogConfigSchedulerAirflow: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-scheduler-automatic-log-config-0" && + .pod == "airflow-scheduler-auto-log-cfg-0" && .container == "airflow" filteredAutomaticLogConfigSchedulerGitSync: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-scheduler-automatic-log-config-0" && + .pod == "airflow-scheduler-auto-log-cfg-0" && .container == "git-sync-0" filteredAutomaticLogConfigSchedulerVector: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-scheduler-automatic-log-config-0" && + .pod == "airflow-scheduler-auto-log-cfg-0" && .container == "vector" filteredCustomLogConfigSchedulerAirflow: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-scheduler-custom-log-config-0" && + .pod == "airflow-scheduler-custom-log-cfg-0" && .container == "airflow" filteredCustomLogConfigSchedulerVector: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-scheduler-custom-log-config-0" && + .pod == "airflow-scheduler-custom-log-cfg-0" && .container == "vector" {% if test_scenario['values']['executor'] == 'celery' %} filteredAutomaticLogConfigWorkerAirflow: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-worker-automatic-log-config-0" && + .pod == "airflow-worker-auto-log-cfg-0" && .container == "airflow" filteredAutomaticLogConfigWorkerGitSync: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-worker-automatic-log-config-0" && + .pod == "airflow-worker-auto-log-cfg-0" && .container == "git-sync-0" filteredAutomaticLogConfigWorkerGitSyncInit: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-worker-automatic-log-config-0" && + .pod == "airflow-worker-auto-log-cfg-0" && .container == "git-sync-0-init" filteredAutomaticLogConfigWorkerVector: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-worker-automatic-log-config-0" && + .pod == "airflow-worker-auto-log-cfg-0" && .container == "vector" filteredCustomLogConfigWorkerAirflow: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-worker-custom-log-config-0" && + .pod == "airflow-worker-custom-log-cfg-0" && .container == "airflow" filteredCustomLogConfigWorkerVector: type: filter inputs: [validEvents] condition: >- - .pod == "airflow-worker-custom-log-config-0" && + .pod == "airflow-worker-custom-log-cfg-0" && .container == "vector" {% elif test_scenario['values']['executor'] == 'kubernetes' %} filteredExampleTriggerTargetDagBashTaskBase: diff --git a/tests/templates/kuttl/resources/30-assert.yaml.j2 b/tests/templates/kuttl/resources/30-assert.yaml.j2 index 3a8e17bf..7c3407e9 100644 --- a/tests/templates/kuttl/resources/30-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/30-assert.yaml.j2 @@ -16,7 +16,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-worker-resources-from-role + name: airflow-worker-from-role spec: template: spec: @@ -40,7 +40,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-worker-resources-from-role-group + name: airflow-worker-from-rg spec: template: spec: @@ -64,7 +64,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: airflow-worker-resources-from-pod-overrides + name: airflow-worker-from-overrides spec: template: spec: diff --git a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 index 0bdcdc99..94190af3 100644 --- a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 @@ -84,9 +84,9 @@ spec: memory: limit: 3001Mi roleGroups: - resources-from-role: + from-role: replicas: 1 - resources-from-role-group: + from-rg: config: resources: cpu: @@ -95,7 +95,7 @@ spec: memory: limit: 3002Mi replicas: 1 - resources-from-pod-overrides: + from-overrides: podOverrides: spec: containers: From e1b22c34951e934358dc4fb7f839b21b53d2d760 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 May 2026 11:29:32 +0200 Subject: [PATCH 14/14] chore: remove REVIEW comments Remove all // REVIEW: comments that were added to aid code review. These served their purpose during the review process and are no longer needed. Co-Authored-By: Claude Opus 4.6 --- rust/operator-binary/src/controller.rs | 12 ------------ rust/operator-binary/src/controller/build.rs | 11 ----------- .../src/controller/build/role_group_builder.rs | 4 ---- rust/operator-binary/src/controller/validate.rs | 3 --- rust/operator-binary/src/controller_commons.rs | 2 -- rust/operator-binary/src/crd/mod.rs | 2 -- rust/operator-binary/src/env_vars.rs | 1 - rust/operator-binary/src/framework/types/operator.rs | 11 ----------- rust/operator-binary/src/main.rs | 1 - rust/operator-binary/src/product_logging.rs | 2 -- rust/operator-binary/src/service.rs | 2 -- 11 files changed, 51 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8ed7a1de..6d47130d 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -159,9 +159,6 @@ impl ValidatedLogging { } } -// REVIEW: ValidatedAirflowCluster is the central validated type. It holds all data needed -// by the build stage so that build() can be infallible. All optional-after-merge fields -// are unwrapped during validation, and logging is pre-validated into ValidatedLogging. #[derive(Clone)] pub struct ValidatedAirflowCluster { metadata: ObjectMeta, @@ -303,12 +300,6 @@ impl ReconcilerError for Error { } } -// REVIEW: The reconcile pipeline is structured as five sequential stages: -// 1. dereference — async, fallible: resolve external references -// 2. validate — sync, fallible: validate and merge configs -// 3. build — sync, infallible: construct Kubernetes resources -// 4. apply — async, fallible: apply resources to the cluster -// 5. update_status — async, fallible: patch status on the CRD pub async fn reconcile( airflow: Arc>, ctx: Arc, @@ -336,9 +327,6 @@ pub async fn reconcile( let validated = validate_cluster(airflow, &dereferenced, &ctx.product_config).context(ValidateSnafu)?; - // REVIEW: build() is infallible — all validation and fallible operations (config - // generation, PodBuilder/ContainerBuilder usage, logging validation) happen in the - // validate stage. The build stage purely assembles Kubernetes resource structs. // --- build (sync, infallible) --- let prepared = build(&validated); diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 952d9605..0875faf5 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -33,9 +33,6 @@ fn main_container_for_role(_role: &AirflowRole) -> Container { Container::Airflow } -// REVIEW: build() is infallible. All validation and fallible operations (config generation, -// PodBuilder/ContainerBuilder usage, logging validation) are performed in the validate -// stage. The build stage purely assembles Kubernetes resource structs from validated data. pub(crate) fn build(validated: &ValidatedAirflowCluster) -> KubernetesResources { let mut stateful_sets = Vec::new(); let mut config_maps = Vec::new(); @@ -140,9 +137,6 @@ fn build_pdb( _ => 1, }); - // REVIEW: from_str_unsafe is used here because the values come from constants (APP_NAME, - // OPERATOR_NAME, AIRFLOW_CONTROLLER_NAME) or validated role names — they are known to be - // valid at compile time or have been validated during the validate stage. Some({ framework::builder::pdb::pod_disruption_budget_builder_with_role( cluster, @@ -240,11 +234,6 @@ fn build_metrics_service( } } -// REVIEW: from_str_unsafe is used for label construction throughout these helpers because -// the inputs are either compile-time constants (APP_NAME, OPERATOR_NAME, etc.) or values -// that have already been validated during the validate stage (role names, role group names, -// product versions). Using from_str_unsafe avoids redundant re-validation in the infallible -// build stage. pub(super) fn build_recommended_labels( cluster: &ValidatedAirflowCluster, role: &str, diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 8e52744e..cd4894e6 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -359,10 +359,6 @@ impl<'a> RoleGroupBuilder<'a> { } fn build_volumes(&self) -> Vec { - // REVIEW: controller_commons::create_volumes is called with the new validated type - // (&ValidatedContainerLogConfigChoice) instead of the old Option<&ContainerLogConfig>. - // This is safe because the logging config has already been validated during the - // validate stage. let mut volumes = controller_commons::create_volumes( &self.rolegroup_ref.object_name(), &self.role_group_config.logging.airflow_container, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a3bcf623..6cf0bd9e 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -793,9 +793,6 @@ fn build_executor_template_config_maps( pb.add_container(airflow_container.build()); pb.add_volumes(airflow.volumes().clone()) .context(AddVolumeSnafu)?; - // REVIEW: controller_commons::create_volumes now takes &ValidatedContainerLogConfigChoice - // instead of Option<&ContainerLogConfig>. The validated type ensures logging config - // has already been checked, so the build stage can use it directly. pb.add_volumes(controller_commons::create_volumes( &executor_rolegroup_ref.object_name(), &validated_config.logging.airflow_container, diff --git a/rust/operator-binary/src/controller_commons.rs b/rust/operator-binary/src/controller_commons.rs index 9cac0007..45766bf4 100644 --- a/rust/operator-binary/src/controller_commons.rs +++ b/rust/operator-binary/src/controller_commons.rs @@ -13,8 +13,6 @@ pub const CONFIG_VOLUME_NAME: &str = "config"; pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; pub const LOG_VOLUME_NAME: &str = "log"; -// REVIEW: parameter changed from Option<&ContainerLogConfig> to &ValidatedContainerLogConfigChoice. -// Pattern matching is simpler now because we match on a flat enum instead of nested Option. pub fn create_volumes( config_map_name: &str, log_config: &ValidatedContainerLogConfigChoice, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3b48d99a..00258006 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -571,8 +571,6 @@ pub struct AirflowOpaConfig { pub cache: UserInformationCache, } -// REVIEW: Ord/PartialOrd added so AirflowRole can be used as a BTreeMap key -// in the new controller's ValidatedAirflowCluster (BTreeMap) #[derive( Clone, Debug, diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index 115cb4d9..c006f4af 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -373,7 +373,6 @@ fn static_envs( /// Return environment variables to be applied to the configuration map used in conjunction with /// the `kubernetesExecutor` worker. -// REVIEW: simplified from &ExecutorConfig — only config.logging.enable_vector_agent was used pub fn build_airflow_template_envs( airflow: &v1alpha2::AirflowCluster, env_overrides: &HashMap, diff --git a/rust/operator-binary/src/framework/types/operator.rs b/rust/operator-binary/src/framework/types/operator.rs index 8e4d3f50..4d060168 100644 --- a/rust/operator-binary/src/framework/types/operator.rs +++ b/rust/operator-binary/src/framework/types/operator.rs @@ -12,7 +12,6 @@ use crate::attributed_string_type; attributed_string_type! { ProductName, "The name of a product", - // REVIEW: example is operator-specific — parameterise when moving to operator-rs "airflow", // A suffix is added to produce a label value. An according compile-time check ensures that // max_length cannot be set higher. @@ -31,13 +30,10 @@ attributed_string_type! { attributed_string_type! { ClusterName, "The name of a cluster/stacklet", - // REVIEW: example is operator-specific — parameterise when moving to operator-rs "my-airflow-cluster", // Suffixes are added to produce resource names. According compile-time checks ensure that // max_length cannot be set higher. Reduced from opensearch's 24 to 22 because airflow's // longest role name ("dagprocessor") is 12 chars vs opensearch's 10. - // REVIEW: max_length is operator-specific (depends on longest RoleName) — parameterise - // when moving to operator-rs (max_length = 22), is_rfc_1035_label_name, is_valid_label_value @@ -46,7 +42,6 @@ attributed_string_type! { attributed_string_type! { ControllerName, "The name of a controller in an operator", - // REVIEW: example is operator-specific — parameterise when moving to operator-rs "airflowcluster", is_valid_label_value } @@ -54,7 +49,6 @@ attributed_string_type! { attributed_string_type! { OperatorName, "The name of an operator", - // REVIEW: example is operator-specific — parameterise when moving to operator-rs "airflow.stackable.tech", is_valid_label_value } @@ -66,8 +60,6 @@ attributed_string_type! { // The role-group name is used to produce resource names. To make sure that all resource names // are valid, max_length is restricted. Compile-time checks ensure that max_length cannot be // set higher if not other names like the RoleName are set lower accordingly. - // REVIEW: max_length is operator-specific (depends on ClusterName and RoleName budgets) — - // parameterise when moving to operator-rs (max_length = 16), is_rfc_1123_label_name, is_valid_label_value @@ -76,13 +68,10 @@ attributed_string_type! { attributed_string_type! { RoleName, "The name of a role name", - // REVIEW: example is operator-specific — parameterise when moving to operator-rs "webserver", // The role name is used to produce resource names. To make sure that all resource names are // valid, max_length is restricted. Compile-time checks ensure that max_length cannot be set // higher if not other names like the RoleGroupName are set lower accordingly. - // REVIEW: max_length is operator-specific (airflow needs 12 for "dagprocessor", opensearch - // uses 10) — parameterise when moving to operator-rs (max_length = 12), is_rfc_1123_label_name, is_valid_label_value diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 85ffe32c..f63266e5 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -177,7 +177,6 @@ async fn main() -> anyhow::Result<()> { ) .graceful_shutdown_on(sigterm_watcher.handle()) .run( - // REVIEW: renamed from reconcile_airflow to reconcile controller::reconcile, controller::error_policy, Arc::new(controller::Ctx { diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 0f56d2af..4bafb481 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -18,8 +18,6 @@ use crate::{ const LOG_CONFIG_FILE: &str = "log_config.py"; const LOG_FILE: &str = "airflow.py.json"; -// REVIEW: signature changed from Logging + two container keys to pre-validated types. -// The function is now infallible — validation happens earlier in the pipeline. /// Extend the ConfigMap with logging and Vector configurations pub fn extend_config_map_with_log_config( rolegroup: &RoleGroupRef, diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index 6b977032..0aaeac61 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -2,8 +2,6 @@ use stackable_operator::{kube::Resource, role_utils::RoleGroupRef}; pub const HEADLESS_SERVICE_SUFFIX: &str = "headless"; -// REVIEW: made generic so the new controller can call this with -// RoleGroupRef instead of RoleGroupRef pub fn stateful_set_service_name( rolegroup_ref: &RoleGroupRef, ) -> Option {