diff --git a/crates/stackable-operator/crds/DummyCluster.yaml b/crates/stackable-operator/crds/DummyCluster.yaml index 18b0000f3..1439afe2b 100644 --- a/crates/stackable-operator/crds/DummyCluster.yaml +++ b/crates/stackable-operator/crds/DummyCluster.yaml @@ -82,6 +82,59 @@ spec: and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. type: boolean type: object + configOverrides: + default: {} + description: |- + Typed config override strategies for Dummy config files. + + Demonstrates both JSON-formatted (`config.json`) and key-value-formatted + (`dummy.properties`) config file overrides. + properties: + config.json: + description: Overrides for the `config.json` file. + nullable: true + oneOf: + - required: + - jsonMergePatch + - required: + - jsonPatches + - required: + - userProvided + properties: + jsonMergePatch: + description: |- + Can be set to arbitrary YAML content, which is converted to JSON and used as + [RFC 7396](https://datatracker.ietf.org/doc/html/rfc7396) JSON merge patch. + type: object + x-kubernetes-preserve-unknown-fields: true + jsonPatches: + description: |- + List of [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902) JSON patches. + + Can be used when more flexibility is needed, e.g. to only modify elements + in a list based on a condition. + + A patch looks something like + + `{"op": "test", "path": "/0/name", "value": "Andrew"}` + + or + + `{"op": "add", "path": "/0/happy", "value": true}` + items: + type: string + type: array + userProvided: + description: Override the entire config file with the specified String. + type: string + type: object + dummy.properties: + additionalProperties: + type: string + description: Overrides for the `dummy.properties` file. + nullable: true + type: object + type: object domainName: description: A validated domain name type conforming to RFC 1123, so e.g. not an IP address type: string @@ -713,17 +766,57 @@ spec: type: object type: object configOverrides: - additionalProperties: - additionalProperties: - type: string - type: object - default: {} description: |- The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. + properties: + config.json: + description: Overrides for the `config.json` file. + nullable: true + oneOf: + - required: + - jsonMergePatch + - required: + - jsonPatches + - required: + - userProvided + properties: + jsonMergePatch: + description: |- + Can be set to arbitrary YAML content, which is converted to JSON and used as + [RFC 7396](https://datatracker.ietf.org/doc/html/rfc7396) JSON merge patch. + type: object + x-kubernetes-preserve-unknown-fields: true + jsonPatches: + description: |- + List of [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902) JSON patches. + + Can be used when more flexibility is needed, e.g. to only modify elements + in a list based on a condition. + + A patch looks something like + + `{"op": "test", "path": "/0/name", "value": "Andrew"}` + + or + + `{"op": "add", "path": "/0/happy", "value": true}` + items: + type: string + type: array + userProvided: + description: Override the entire config file with the specified String. + type: string + type: object + dummy.properties: + additionalProperties: + type: string + description: Overrides for the `dummy.properties` file. + nullable: true + type: object type: object envOverrides: additionalProperties: @@ -1325,17 +1418,57 @@ spec: type: object type: object configOverrides: - additionalProperties: - additionalProperties: - type: string - type: object - default: {} description: |- The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. + properties: + config.json: + description: Overrides for the `config.json` file. + nullable: true + oneOf: + - required: + - jsonMergePatch + - required: + - jsonPatches + - required: + - userProvided + properties: + jsonMergePatch: + description: |- + Can be set to arbitrary YAML content, which is converted to JSON and used as + [RFC 7396](https://datatracker.ietf.org/doc/html/rfc7396) JSON merge patch. + type: object + x-kubernetes-preserve-unknown-fields: true + jsonPatches: + description: |- + List of [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902) JSON patches. + + Can be used when more flexibility is needed, e.g. to only modify elements + in a list based on a condition. + + A patch looks something like + + `{"op": "test", "path": "/0/name", "value": "Andrew"}` + + or + + `{"op": "add", "path": "/0/happy", "value": true}` + items: + type: string + type: array + userProvided: + description: Override the entire config file with the specified String. + type: string + type: object + dummy.properties: + additionalProperties: + type: string + description: Overrides for the `dummy.properties` file. + nullable: true + type: object type: object envOverrides: additionalProperties: diff --git a/crates/stackable-operator/src/commons/resources.rs b/crates/stackable-operator/src/commons/resources.rs index 1eac198a9..ccaf03ec2 100644 --- a/crates/stackable-operator/src/commons/resources.rs +++ b/crates/stackable-operator/src/commons/resources.rs @@ -30,6 +30,9 @@ //! role_utils::Role, //! }; //! +//! #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] +//! pub struct ProductConfigOverrides {} +//! //! #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] //! #[kube( //! group = "product.stackable.tech", @@ -46,7 +49,7 @@ //! #[serde(rename_all = "camelCase")] //! pub struct ProductSpec { //! #[serde(default, skip_serializing_if = "Option::is_none")] -//! pub nodes: Option>, +//! pub nodes: Option>, //! } //! //! #[derive(Debug, Default, PartialEq, Fragment, JsonSchema)] diff --git a/crates/stackable-operator/src/config_overrides.rs b/crates/stackable-operator/src/config_overrides.rs new file mode 100644 index 000000000..0cbc63eeb --- /dev/null +++ b/crates/stackable-operator/src/config_overrides.rs @@ -0,0 +1,284 @@ +//! Building-block types for strategy-based `configOverrides`. +//! +//! Operators declare typed override structs choosing patch strategies per file +//! (e.g. [`JsonConfigOverrides`] for JSON files, [`KeyValueConfigOverrides`] for +//! properties files). The types here are composed by each operator into its +//! CRD-specific `configOverrides` struct. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, Snafu}; + +use crate::utils::crds::raw_object_schema; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to serialize base document to JSON"))] + SerializeBaseDocument { source: serde_json::Error }, + + #[snafu(display("failed to apply JSON patch (RFC 6902)"))] + ApplyJsonPatch { source: json_patch::PatchError }, + + #[snafu(display("failed to deserialize JSON patch operation {index} from string"))] + DeserializeJsonPatchOperation { + source: serde_json::Error, + index: usize, + }, + + #[snafu(display("failed to parse user-provided JSON content"))] + ParseUserProvidedJson { source: serde_json::Error }, +} + +/// Trait that allows the product config pipeline to extract flat key-value +/// overrides from any `configOverrides` type. +/// +/// Typed override structs that have no key-value files can use the default +/// implementation, which returns an empty map. +pub trait KeyValueOverridesProvider { + fn get_key_value_overrides(&self, _file: &str) -> BTreeMap> { + BTreeMap::new() + } +} + +/// Flat key-value overrides for `*.properties`, Hadoop XML, etc. +/// +/// This is backwards-compatible with the existing flat key-value YAML format +/// used by `HashMap`. +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct KeyValueConfigOverrides { + #[serde(flatten)] + pub overrides: BTreeMap, +} + +impl KeyValueConfigOverrides { + /// Returns the overrides as a `BTreeMap>`, matching + /// the format expected by the product config pipeline. + /// + /// This is useful when implementing [`KeyValueOverridesProvider`] for a + /// typed override struct that contains [`KeyValueConfigOverrides`] fields. + pub fn as_overrides(&self) -> BTreeMap> { + self.overrides + .iter() + .map(|(k, v)| (k.clone(), Some(v.clone()))) + .collect() + } +} + +/// ConfigOverrides that can be applied to a JSON file. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum JsonConfigOverrides { + /// Can be set to arbitrary YAML content, which is converted to JSON and used as + /// [RFC 7396](https://datatracker.ietf.org/doc/html/rfc7396) JSON merge patch. + #[schemars(schema_with = "raw_object_schema")] + JsonMergePatch(serde_json::Value), + + /// List of [RFC 6902](https://datatracker.ietf.org/doc/html/rfc6902) JSON patches. + /// + /// Can be used when more flexibility is needed, e.g. to only modify elements + /// in a list based on a condition. + /// + /// A patch looks something like + /// + /// `{"op": "test", "path": "/0/name", "value": "Andrew"}` + /// + /// or + /// + /// `{"op": "add", "path": "/0/happy", "value": true}` + JsonPatches(Vec), + + /// Override the entire config file with the specified String. + UserProvided(String), +} + +impl JsonConfigOverrides { + /// Applies this override to a base JSON document and returns the patched + /// document as a [`serde_json::Value`]. + /// + /// For [`JsonConfigOverrides::JsonMergePatch`] and + /// [`JsonConfigOverrides::JsonPatches`], the base document is patched + /// according to the respective RFC. + /// + /// For [`JsonConfigOverrides::UserProvided`], the base document is ignored + /// entirely and the user-provided string is parsed and returned. + pub fn apply(&self, base: &serde_json::Value) -> Result { + match self { + JsonConfigOverrides::JsonMergePatch(patch) => { + let mut doc = base.clone(); + json_patch::merge(&mut doc, patch); + Ok(doc) + } + JsonConfigOverrides::JsonPatches(patches) => { + let mut doc = base.clone(); + let operations: Vec = patches + .iter() + .enumerate() + .map(|(index, patch_str)| { + serde_json::from_str(patch_str) + .context(DeserializeJsonPatchOperationSnafu { index }) + }) + .collect::, _>>()?; + json_patch::patch(&mut doc, &operations).context(ApplyJsonPatchSnafu)?; + Ok(doc) + } + JsonConfigOverrides::UserProvided(content) => { + serde_json::from_str(content).context(ParseUserProvidedJsonSnafu) + } + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn json_merge_patch_add_and_overwrite_fields() { + let base = json!({ + "bundles": { + "authz": { + "polling": { + "min_delay_seconds": 10, + "max_delay_seconds": 20 + } + } + } + }); + + let overrides = JsonConfigOverrides::JsonMergePatch(json!({ + "bundles": { + "authz": { + "polling": { + "min_delay_seconds": 3, + "max_delay_seconds": 5 + } + } + }, + "default_decision": "/http/example/authz/allow" + })); + + let result = overrides.apply(&base).expect("merge patch should succeed"); + + assert_eq!( + result["bundles"]["authz"]["polling"]["min_delay_seconds"], + 3 + ); + assert_eq!( + result["bundles"]["authz"]["polling"]["max_delay_seconds"], + 5 + ); + assert_eq!(result["default_decision"], "/http/example/authz/allow"); + } + + #[test] + fn json_merge_patch_remove_field_with_null() { + let base = json!({ + "keep": "this", + "remove": "this" + }); + + let overrides = JsonConfigOverrides::JsonMergePatch(json!({ + "remove": null + })); + + let result = overrides.apply(&base).expect("merge patch should succeed"); + + assert_eq!(result["keep"], "this"); + assert!(result.get("remove").is_none()); + } + + #[test] + fn json_patch_add_remove_replace() { + let base = json!({ + "foo": "bar", + "baz": "qux" + }); + + let overrides = JsonConfigOverrides::JsonPatches(vec![ + r#"{"op": "replace", "path": "/foo", "value": "replaced"}"#.to_owned(), + r#"{"op": "remove", "path": "/baz"}"#.to_owned(), + r#"{"op": "add", "path": "/new_key", "value": "new_value"}"#.to_owned(), + ]); + + let result = overrides.apply(&base).expect("JSON patch should succeed"); + + assert_eq!(result["foo"], "replaced"); + assert!(result.get("baz").is_none()); + assert_eq!(result["new_key"], "new_value"); + } + + #[test] + fn json_patch_invalid_path_returns_error() { + let base = json!({"foo": "bar"}); + + let overrides = JsonConfigOverrides::JsonPatches(vec![ + r#"{"op": "remove", "path": "/nonexistent"}"#.to_owned(), + ]); + + let result = overrides.apply(&base); + assert!( + matches!(result.unwrap_err(), Error::ApplyJsonPatch { source } if source.to_string() + == "operation '/0' failed at path '/nonexistent': path is invalid"), + "removing a nonexistent path should fail" + ); + } + + #[test] + fn json_patch_invalid_operation_returns_error() { + let base = json!({"foo": "bar"}); + + let overrides = JsonConfigOverrides::JsonPatches(vec![r#"{"not_an_op": true}"#.to_owned()]); + + let result = overrides.apply(&base); + assert!( + matches!(result.unwrap_err(), Error::DeserializeJsonPatchOperation { source, index: 0 } if source.to_string() + == "missing field `op` at line 1 column 19"), + "invalid patch operation should return an error" + ); + } + + #[test] + fn user_provided_ignores_base() { + let base = json!({"foo": "bar"}); + let content = "{\"custom\": true}"; + + let overrides = JsonConfigOverrides::UserProvided(content.to_owned()); + + let result = overrides + .apply(&base) + .expect("user provided should succeed"); + assert_eq!(result, json!({"custom": true})); + } + + #[test] + fn user_provided_invalid_json_returns_error() { + let base = json!({"foo": "bar"}); + + let overrides = JsonConfigOverrides::UserProvided("not valid json".to_owned()); + + let result = overrides.apply(&base); + assert!( + matches!(result.unwrap_err(), Error::ParseUserProvidedJson { source } if source.to_string() + == "expected ident at line 1 column 2"), + "invalid JSON should return an error" + ); + } + + #[test] + fn key_value_config_overrides_as_overrides() { + let mut overrides = BTreeMap::new(); + overrides.insert("key1".to_owned(), "value1".to_owned()); + overrides.insert("key2".to_owned(), "value2".to_owned()); + + let kv = KeyValueConfigOverrides { overrides }; + let result = kv.as_overrides(); + + assert_eq!(result.len(), 2); + assert_eq!(result.get("key1"), Some(&Some("value1".to_owned()))); + assert_eq!(result.get("key2"), Some(&Some("value2".to_owned()))); + } +} diff --git a/crates/stackable-operator/src/lib.rs b/crates/stackable-operator/src/lib.rs index 46616252c..1e0fe620c 100644 --- a/crates/stackable-operator/src/lib.rs +++ b/crates/stackable-operator/src/lib.rs @@ -12,6 +12,7 @@ pub mod client; pub mod cluster_resources; pub mod commons; pub mod config; +pub mod config_overrides; pub mod constants; pub mod cpu; #[cfg(feature = "crds")] diff --git a/crates/stackable-operator/src/product_config_utils.rs b/crates/stackable-operator/src/product_config_utils.rs index bcf75572e..67c43fdeb 100644 --- a/crates/stackable-operator/src/product_config_utils.rs +++ b/crates/stackable-operator/src/product_config_utils.rs @@ -7,7 +7,10 @@ use serde::Serialize; use snafu::{ResultExt, Snafu}; use tracing::{debug, error, warn}; -use crate::role_utils::{CommonConfiguration, Role}; +use crate::{ + config_overrides::KeyValueOverridesProvider, + role_utils::{CommonConfiguration, Role}, +}; pub const CONFIG_OVERRIDE_FILE_HEADER_KEY: &str = "EXPERIMENTAL_FILE_HEADER"; pub const CONFIG_OVERRIDE_FILE_FOOTER_KEY: &str = "EXPERIMENTAL_FILE_FOOTER"; @@ -171,20 +174,21 @@ pub fn config_for_role_and_group<'a>( /// - `roles`: A map keyed by role names. The value is a tuple of a vector of `PropertyNameKind` /// like (Cli, Env or Files) and [`crate::role_utils::Role`] with a boxed [`Configuration`]. #[allow(clippy::type_complexity)] -pub fn transform_all_roles_to_config( - resource: &T::Configurable, +pub fn transform_all_roles_to_config( + resource: &Config::Configurable, roles: HashMap< String, ( Vec, - Role, + Role, ), >, ) -> Result where - T: Configuration, - U: Default + JsonSchema + Serialize, - ProductSpecificCommonConfig: Default + JsonSchema + Serialize, + Config: Configuration, + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize + KeyValueOverridesProvider, { let mut result = HashMap::new(); @@ -380,16 +384,17 @@ fn process_validation_result( /// - `role_name` - The name of the role. /// - `role` - The role for which to transform the configuration parameters. /// - `property_kinds` - Used as "buckets" to partition the configuration properties by. -fn transform_role_to_config( - resource: &T::Configurable, +fn transform_role_to_config( + resource: &Config::Configurable, role_name: &str, - role: &Role, + role: &Role, property_kinds: &[PropertyNameKind], ) -> Result where - T: Configuration, - U: Default + JsonSchema + Serialize, - ProductSpecificCommonConfig: Default + JsonSchema + Serialize, + Config: Configuration, + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize + KeyValueOverridesProvider, { let mut result = HashMap::new(); @@ -444,14 +449,14 @@ where /// - `role_name` - Not used directly but passed on to the `Configuration::compute_*` calls. /// - `config` - The configuration properties to partition. /// - `property_kinds` - The "buckets" used to partition the configuration properties. -fn parse_role_config( - resource: &::Configurable, +fn parse_role_config( + resource: &::Configurable, role_name: &str, - config: &CommonConfiguration, + config: &CommonConfiguration, property_kinds: &[PropertyNameKind], ) -> Result>>> where - T: Configuration, + Config: Configuration, { let mut result = HashMap::new(); for property_kind in property_kinds { @@ -474,12 +479,13 @@ where Ok(result) } -fn parse_role_overrides( - config: &CommonConfiguration, +fn parse_role_overrides( + config: &CommonConfiguration, property_kinds: &[PropertyNameKind], ) -> Result>>> where - T: Configuration, + Config: Configuration, + ConfigOverrides: KeyValueOverridesProvider, { let mut result = HashMap::new(); for property_kind in property_kinds { @@ -511,23 +517,15 @@ where Ok(result) } -fn parse_file_overrides( - config: &CommonConfiguration, +fn parse_file_overrides( + config: &CommonConfiguration, file: &str, ) -> Result>> where - T: Configuration, + Config: Configuration, + ConfigOverrides: KeyValueOverridesProvider, { - let mut final_overrides: BTreeMap> = BTreeMap::new(); - - // For Conf files only process overrides that match our file name - if let Some(config) = config.config_overrides.get(file) { - for (key, value) in config { - final_overrides.insert(key.clone(), Some(value.clone())); - } - } - - Ok(final_overrides) + Ok(config.config_overrides.get_key_value_overrides(file)) } /// Extract the environment variables of a rolegroup config into a vector of EnvVars. @@ -591,7 +589,7 @@ pub fn env_vars_from_rolegroup_config( /// product_config_utils::env_vars_from, role_utils::CommonConfiguration, /// }; /// -/// let common_config = CommonConfiguration::<(), ()> { +/// let common_config = CommonConfiguration::<(), (), ()> { /// env_overrides: [("VAR".to_string(), "value".to_string())] /// .into_iter() /// .collect(), @@ -693,9 +691,30 @@ mod tests { use k8s_openapi::api::core::v1::PodTemplateSpec; use rstest::*; + use schemars::JsonSchema; + use serde::{Deserialize, Serialize}; use super::*; - use crate::role_utils::{GenericProductSpecificCommonConfig, Role, RoleGroup}; + use crate::{ + config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, + role_utils::{GenericCommonConfig, Role, RoleGroup}, + }; + + /// Test-only config overrides type that wraps per-file key-value overrides. + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + struct TestConfigOverrides { + #[serde(flatten)] + files: HashMap, + } + + impl KeyValueOverridesProvider for TestConfigOverrides { + fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { + self.files + .get(file) + .map(|kv| kv.as_overrides()) + .unwrap_or_default() + } + } const ROLE_GROUP: &str = "role_group"; @@ -776,27 +795,24 @@ mod tests { fn build_common_config( test_config: Option>, - config_overrides: Option>>, + config_overrides: Option, env_overrides: Option>, cli_overrides: Option>, - ) -> CommonConfiguration, GenericProductSpecificCommonConfig> { + ) -> CommonConfiguration, GenericCommonConfig, TestConfigOverrides> { CommonConfiguration { config: test_config.unwrap_or_default(), config_overrides: config_overrides.unwrap_or_default(), env_overrides: env_overrides.unwrap_or_default(), cli_overrides: cli_overrides.unwrap_or_default(), pod_overrides: PodTemplateSpec::default(), - product_specific_common_config: GenericProductSpecificCommonConfig::default(), + product_specific_common_config: GenericCommonConfig::default(), } } - fn build_config_override( - file_name: &str, - property: &str, - ) -> Option>> { - Some( - collection!( file_name.to_string() => collection!( property.to_string() => property.to_string())), - ) + fn build_config_override(file_name: &str, property: &str) -> Option { + Some(TestConfigOverrides { + files: collection!( file_name.to_string() => KeyValueConfigOverrides { overrides: collection!( property.to_string() => property.to_string()) }), + }) } fn build_env_override(property: &str) -> Option> { @@ -812,7 +828,7 @@ mod tests { group_config: bool, role_overrides: bool, group_overrides: bool, - ) -> Role, TestRoleConfig> { + ) -> Role, TestConfigOverrides, TestRoleConfig> { let role_group = ROLE_GROUP.to_string(); let file_name = "foo.conf"; @@ -1245,16 +1261,16 @@ mod tests { BTreeMap::from([ ("cli".to_string(), GROUP_CLI_OVERRIDE.to_string()), ]), - HashMap::from([ - ("file".to_string(), HashMap::from([ + TestConfigOverrides { files: HashMap::from([ + ("file".to_string(), KeyValueConfigOverrides { overrides: BTreeMap::from([ ("file".to_string(), ROLE_CONF_OVERRIDE.to_string()) - ])) - ]), - HashMap::from([ - ("file".to_string(), HashMap::from([ + ]) }) + ]) }, + TestConfigOverrides { files: HashMap::from([ + ("file".to_string(), KeyValueConfigOverrides { overrides: BTreeMap::from([ ("file".to_string(), GROUP_CONF_OVERRIDE.to_string()) - ])) - ]), + ]) }) + ]) }, collection ! { ROLE_GROUP.to_string() => collection ! { PropertyNameKind::Env => collection ! { @@ -1278,12 +1294,12 @@ mod tests { ("cli".to_string(), ROLE_CLI_OVERRIDE.to_string()), ]), BTreeMap::from([]), - HashMap::from([ - ("file".to_string(), HashMap::from([ + TestConfigOverrides { files: HashMap::from([ + ("file".to_string(), KeyValueConfigOverrides { overrides: BTreeMap::from([ ("file".to_string(), ROLE_CONF_OVERRIDE.to_string()) - ])) - ]), - HashMap::from([]), + ]) }) + ]) }, + TestConfigOverrides::default(), collection ! { ROLE_GROUP.to_string() => collection ! { PropertyNameKind::Env => collection ! { @@ -1303,8 +1319,8 @@ mod tests { HashMap::from([]), BTreeMap::from([]), BTreeMap::from([]), - HashMap::from([]), - HashMap::from([]), + TestConfigOverrides::default(), + TestConfigOverrides::default(), collection ! { ROLE_GROUP.to_string() => collection ! { PropertyNameKind::Env => collection ! { @@ -1324,14 +1340,14 @@ mod tests { #[case] group_env_override: HashMap, #[case] role_cli_override: BTreeMap, #[case] group_cli_override: BTreeMap, - #[case] role_conf_override: HashMap>, - #[case] group_conf_override: HashMap>, + #[case] role_conf_override: TestConfigOverrides, + #[case] group_conf_override: TestConfigOverrides, #[case] expected: HashMap< String, HashMap>>, >, ) { - let role: Role, TestRoleConfig> = Role { + let role: Role, TestConfigOverrides, TestRoleConfig> = Role { config: build_common_config( build_test_config(ROLE_CONFIG, ROLE_ENV, ROLE_CLI), Some(role_conf_override), @@ -1428,7 +1444,10 @@ mod tests { #[allow(clippy::type_complexity)] let roles: HashMap< String, - (Vec, Role, TestRoleConfig>), + ( + Vec, + Role, TestConfigOverrides, TestRoleConfig>, + ), > = collection! { role_1.to_string() => (vec![PropertyNameKind::File(file_name.to_string()), PropertyNameKind::Env], Role { config: build_common_config( @@ -1525,7 +1544,10 @@ mod tests { #[allow(clippy::type_complexity)] let roles: HashMap< String, - (Vec, Role, TestRoleConfig>), + ( + Vec, + Role, TestConfigOverrides, TestRoleConfig>, + ), > = collection! { role_1.to_string() => (vec![PropertyNameKind::File(file_name.to_string()), PropertyNameKind::Env], Role { config: CommonConfiguration::default(), diff --git a/crates/stackable-operator/src/role_utils.rs b/crates/stackable-operator/src/role_utils.rs index b82bd6623..4b0984b24 100644 --- a/crates/stackable-operator/src/role_utils.rs +++ b/crates/stackable-operator/src/role_utils.rs @@ -118,16 +118,19 @@ pub enum Error { #[serde( rename_all = "camelCase", bound( - deserialize = "T: Default + Deserialize<'de>, ProductSpecificCommonConfig: Default + Deserialize<'de>" + deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Default + Deserialize<'de>, ConfigOverrides: Default + Deserialize<'de>" ) )] -pub struct CommonConfiguration { +#[schemars( + bound = "Config: JsonSchema, CommonConfig: JsonSchema, ConfigOverrides: Default + JsonSchema" +)] +pub struct CommonConfiguration { #[serde(default)] - // We can't depend on T being `Default`, since that trait is not object-safe + // We can't depend on Config being `Default`, since that trait is not object-safe // We only need to generate schemas for fully specified types, but schemars_derive // does not support specifying custom bounds. #[schemars(default = "Self::default_config")] - pub config: T, + pub config: Config, /// The `configOverrides` can be used to configure properties in product config files /// that are not exposed in the CRD. Read the @@ -135,7 +138,7 @@ pub struct CommonConfiguration { /// and consult the operator specific usage guide documentation for details on the /// available config files and settings for the specific product. #[serde(default)] - pub config_overrides: HashMap>, + pub config_overrides: ConfigOverrides, /// `envOverrides` configure environment variables to be set in the Pods. /// It is a map from strings to strings - environment variables and the value to set. @@ -168,17 +171,19 @@ pub struct CommonConfiguration { // If [`JavaCommonConfig`] is used, please use [`Role::get_merged_jvm_argument_overrides`] instead of // reading this field directly. #[serde(flatten, default)] - pub product_specific_common_config: ProductSpecificCommonConfig, + pub product_specific_common_config: CommonConfig, } -impl CommonConfiguration { +impl + CommonConfiguration +{ fn default_config() -> serde_json::Value { serde_json::json!({}) } } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] -pub struct GenericProductSpecificCommonConfig {} +pub struct GenericCommonConfig {} #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] @@ -291,43 +296,47 @@ impl JvmArgumentOverrides { // Everything below is only a "normal" comment, not rustdoc - so we don't bloat the CRD documentation // with technical (Rust) details. // -// `T` here is the `config` shared between role and roleGroup. +// `Config` here is the `config` shared between role and roleGroup. // -// `U` here is the `roleConfig` only available on the role. It defaults to [`GenericRoleConfig`], which is +// `RoleConfig` here is the `roleConfig` only available on the role. It defaults to [`GenericRoleConfig`], which is // sufficient for most of the products. There are some exceptions, where e.g. [`EmptyRoleConfig`] is used. // However, product-operators can define their own - custom - struct and use that here. #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct Role< - T, - U = GenericRoleConfig, - ProductSpecificCommonConfig = GenericProductSpecificCommonConfig, + Config, + ConfigOverrides, + RoleConfig = GenericRoleConfig, + CommonConfig = GenericCommonConfig, > where // Don't remove this trait bounds!!! // We don't know why, but if you remove either of them, the generated default value in the CRDs will // be missing! - U: Default + JsonSchema + Serialize, - ProductSpecificCommonConfig: Default + JsonSchema + Serialize, + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, { #[serde( flatten, bound( - deserialize = "T: Default + Deserialize<'de>, ProductSpecificCommonConfig: Deserialize<'de>" + deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Deserialize<'de>, ConfigOverrides: Deserialize<'de>" ) )] - pub config: CommonConfiguration, + pub config: CommonConfiguration, #[serde(default)] - pub role_config: U, + pub role_config: RoleConfig, - pub role_groups: HashMap>, + pub role_groups: HashMap>, } -impl Role +impl + Role where - T: Configuration + 'static, - U: Default + JsonSchema + Serialize, - ProductSpecificCommonConfig: Default + JsonSchema + Serialize + Clone, + Config: Configuration + 'static, + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize + Clone, + ConfigOverrides: Default + JsonSchema + Serialize, { /// This casts a generic struct implementing [`crate::product_config_utils::Configuration`] /// and used in [`Role`] into a Box of a dynamically dispatched @@ -336,12 +345,16 @@ where /// have different structs implementing Configuration. pub fn erase( self, - ) -> Role>, U, ProductSpecificCommonConfig> - { + ) -> Role< + Box>, + ConfigOverrides, + RoleConfig, + CommonConfig, + > { Role { config: CommonConfiguration { config: Box::new(self.config.config) - as Box>, + as Box>, config_overrides: self.config.config_overrides, env_overrides: self.config.env_overrides, cli_overrides: self.config.cli_overrides, @@ -358,7 +371,7 @@ where RoleGroup { config: CommonConfiguration { config: Box::new(group.config.config) - as Box>, + as Box>, config_overrides: group.config.config_overrides, env_overrides: group.config.env_overrides, cli_overrides: group.config.cli_overrides, @@ -376,9 +389,11 @@ where } } -impl Role +impl + Role where - U: Default + JsonSchema + Serialize, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, { /// Merges jvm argument overrides from /// @@ -431,25 +446,30 @@ pub struct EmptyRoleConfig {} #[serde( rename_all = "camelCase", bound( - deserialize = "T: Default + Deserialize<'de>, ProductSpecificCommonConfig: Default + Deserialize<'de>" + deserialize = "Config: Default + Deserialize<'de>, CommonConfig: Default + Deserialize<'de>, ConfigOverrides: Default + Deserialize<'de>" ) )] -pub struct RoleGroup { +#[schemars( + bound = "Config: JsonSchema, CommonConfig: JsonSchema, ConfigOverrides: Default + JsonSchema" +)] +pub struct RoleGroup { #[serde(flatten)] - pub config: CommonConfiguration, + pub config: CommonConfiguration, pub replicas: Option, } -impl RoleGroup { - pub fn validate_config( +impl RoleGroup { + pub fn validate_config( &self, - role: &Role, - default_config: &T, + role: &Role, + default_config: &Config, ) -> Result where - C: FromFragment, - T: Merge + Clone, - U: Default + JsonSchema + Serialize, + C: FromFragment, + Config: Merge + Clone, + RoleConfig: Default + JsonSchema + Serialize, + CommonConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, { let mut role_config = role.config.config.clone(); role_config.merge(default_config); @@ -511,6 +531,9 @@ mod tests { use super::*; use crate::role_utils::JavaCommonConfig; + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + struct EmptyConfigOverrides {} + #[test] fn test_merge_java_common_config() { // The operator generates some JVM arguments @@ -527,7 +550,7 @@ mod tests { .into(), ); - let entire_role: Role<(), GenericRoleConfig, JavaCommonConfig> = + let entire_role: Role<(), EmptyConfigOverrides, GenericRoleConfig, JavaCommonConfig> = serde_yaml::from_str(" # Let's say we want to set some additional HTTP Proxy and IPv4 settings # And we don't like the garbage collector for some reason... @@ -589,8 +612,9 @@ mod tests { let operator_generated = JvmArgumentOverrides::new_with_only_additions(["-Xms1m".to_owned()].into()); - let entire_role: Role<(), GenericRoleConfig, JavaCommonConfig> = serde_yaml::from_str( - " + let entire_role: Role<(), EmptyConfigOverrides, GenericRoleConfig, JavaCommonConfig> = + serde_yaml::from_str( + " jvmArgumentOverrides: add: - -Xms2m @@ -600,8 +624,8 @@ mod tests { add: - -Xms3m ", - ) - .expect("Failed to parse role"); + ) + .expect("Failed to parse role"); let merged_jvm_argument_overrides = entire_role .get_merged_jvm_argument_overrides("default", &operator_generated) diff --git a/crates/xtask/src/crd/dummy.rs b/crates/xtask/src/crd/dummy.rs index 28ed0581c..7edef1043 100644 --- a/crates/xtask/src/crd/dummy.rs +++ b/crates/xtask/src/crd/dummy.rs @@ -1,7 +1,10 @@ +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{JvmHeapLimits, Resources}, config::fragment::Fragment, + config_overrides::{JsonConfigOverrides, KeyValueConfigOverrides, KeyValueOverridesProvider}, crd::git_sync::v1alpha2::GitSync, deep_merger::ObjectOverrides, kube::CustomResource, @@ -12,6 +15,44 @@ use stackable_operator::{ }; use strum::EnumIter; +/// Typed config override strategies for Dummy config files. +/// +/// Demonstrates both JSON-formatted (`config.json`) and key-value-formatted +/// (`dummy.properties`) config file overrides. +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[schemars(crate = "stackable_operator::schemars")] +#[serde(rename_all = "camelCase")] +pub struct DummyConfigOverrides { + /// Overrides for the `config.json` file. + #[serde( + default, + rename = "config.json", + skip_serializing_if = "Option::is_none" + )] + pub config_json: Option, + + /// Overrides for the `dummy.properties` file. + #[serde( + default, + rename = "dummy.properties", + skip_serializing_if = "Option::is_none" + )] + pub dummy_properties: Option, +} + +impl KeyValueOverridesProvider for DummyConfigOverrides { + fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { + match file { + "dummy.properties" => self + .dummy_properties + .as_ref() + .map(|o| o.as_overrides()) + .unwrap_or_default(), + _ => BTreeMap::new(), + } + } +} + #[versioned( version(name = "v1alpha1"), crates( @@ -33,7 +74,7 @@ pub mod versioned { #[schemars(crate = "stackable_operator::schemars")] #[serde(rename_all = "camelCase")] pub struct DummyClusterSpec { - nodes: Option>, + nodes: Option>, // Not versioned yet stackable_affinity: stackable_operator::commons::affinity::StackableAffinity, @@ -54,6 +95,9 @@ pub mod versioned { #[serde(default)] pub object_overrides: ObjectOverrides, + #[serde(default)] + config_overrides: DummyConfigOverrides, + // Already versioned client_authentication_details: stackable_operator::crd::authentication::core::v1alpha1::ClientAuthenticationDetails,