From 54f574bc802488ad270f03182f2b364aac547f35 Mon Sep 17 00:00:00 2001 From: akarsh vidyarthi <168165668+akarsh11111111@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:56:07 +0530 Subject: [PATCH] feat: allow disabling containerdebug for SparkApplications Closes #730 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 + .../usage-guide/operations/applications.adoc | 22 +++++ extra/crds.yaml | 18 ++++ rust/operator-binary/src/crd/mod.rs | 95 ++++++++++++++----- .../src/crd/template_merger.rs | 47 +++++++++ .../src/spark_k8s_controller/build/pod.rs | 60 ++++++++++++ 6 files changed, 222 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dceab257..26c26bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- Add a `containerDebug.enabled` opt-out for SparkApplication submit, driver, and executor containers ([#730]). - Add support for emitting [OpenLineage](https://openlineage.io/) events from `SparkApplications` via `spec.lineage` ([#726]). - Support for Spark `4.1.3` and `4.2.0` ([#790]). - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field @@ -90,6 +91,7 @@ All notable changes to this project will be documented in this file. - The submit Job and the history server no longer print S3 credentials to the container log ([#796]). [#721]: https://github.com/stackabletech/spark-k8s-operator/pull/721 +[#730]: https://github.com/stackabletech/spark-k8s-operator/issues/730 [#726]: https://github.com/stackabletech/spark-k8s-operator/pull/726 [#727]: https://github.com/stackabletech/spark-k8s-operator/pull/727 [#732]: https://github.com/stackabletech/spark-k8s-operator/pull/732 diff --git a/docs/modules/spark-k8s/pages/usage-guide/operations/applications.adoc b/docs/modules/spark-k8s/pages/usage-guide/operations/applications.adoc index 753b0d99..3f0cb8ca 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/operations/applications.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/operations/applications.adoc @@ -23,3 +23,25 @@ Application driver pods expose Prometheus metrics at the following endpoints: * `/metrics/executors/prometheus` for executor instances. These endpoints are available on the same port as the Spark UI, which is 4040 by default. + +== Container diagnostics + +The `containerdebug` process runs by default in the submit, driver, and executor containers. To +disable it for a SparkApplication, set `spec.containerDebug.enabled` to `false`: + +[source,yaml] +---- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: spark-example +spec: + mode: cluster + mainApplicationFile: local:///path/to/application.py + sparkImage: + productVersion: 3.5.8 + containerDebug: + enabled: false +---- + +Omitting `containerDebug` or `enabled` keeps the process enabled. diff --git a/extra/crds.yaml b/extra/crds.yaml index 0c771bb2..b6835cc6 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -33,6 +33,15 @@ spec: items: type: string type: array + containerDebug: + description: Configure the containerdebug process for this application. It is enabled by default. + nullable: true + properties: + enabled: + default: true + description: Whether to run containerdebug. Defaults to `true`. + type: boolean + type: object deps: default: excludePackages: [] @@ -4865,6 +4874,15 @@ spec: items: type: string type: array + containerDebug: + description: Configure the containerdebug process for this application. It is enabled by default. + nullable: true + properties: + enabled: + default: true + description: Whether to run containerdebug. Defaults to `true`. + type: boolean + type: object deps: default: excludePackages: [] diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 15f33085..cd17dfae 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -209,6 +209,10 @@ pub mod versioned { #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, + /// Configure the containerdebug process for this application. It is enabled by default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub container_debug: Option, + /// The job builds a spark-submit command, complete with arguments and referenced dependencies /// such as templates, and passes it on to Spark. /// The reason this property uses its own type (SubmitConfigFragment) is because logging is not @@ -273,6 +277,19 @@ pub mod versioned { pub lineage: Option, } + /// Configuration for the containerdebug process. + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct ContainerDebugConfig { + /// Whether to run containerdebug. Defaults to `true`. + #[serde(default = "default_true")] + pub enabled: bool, + } + + fn default_true() -> bool { + true + } + /// OpenLineage configuration of a `SparkApplication`: the shared OpenLineage config, plus /// Spark specific settings. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -867,16 +884,29 @@ impl v1alpha1::SparkApplication { submit_cmd.extend(self.spec.args.clone()); - Ok(vec![ - format!( + let mut commands = Vec::new(); + if self.container_debug_enabled() { + commands.push(format!( "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &" - ), + )); + } + commands.extend([ build_truststore_commands, // xtrace prints expanded credentials, `--verbose` prints the config redacted "set +x".to_string(), "echo 'Running \"spark-submit [...]\", which is not shown as it can contain credentials'".to_string(), submit_cmd.join(" "), - ]) + ]); + + Ok(commands) + } + + fn container_debug_enabled(&self) -> bool { + self.spec + .container_debug + .as_ref() + .map(|config| config.enabled) + .unwrap_or(true) } /// The base environment for the submit, driver and executor containers: the user-defined @@ -902,24 +932,20 @@ impl v1alpha1::SparkApplication { .context(ParseSparkEnvVarNameSnafu)?; } - // These env variables enable the `containerdebug` process in driver and executor pods. - // More precisely, this process runs in the background of every `spark` container. - // - `CONTAINERDEBUG_LOG_DIRECTORY` - is the location where tracing information from the process - // is written. This directory is created by the process itself. - // - `_STACKABLE_PRE_HOOK` - is evaluated by the entrypoint script (run-spark.sh) in the Spark images - // before the actual JVM process is started. The result of this evaluation is that the - // `containerdebug` process is executed in the background. - env = env - .with_value( - &CONTAINERDEBUG_LOG_DIRECTORY, - format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), - ) - .with_value( - &STACKABLE_PRE_HOOK, - format!( - "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &" - ), - ); + if self.container_debug_enabled() { + // These env variables enable the `containerdebug` process in driver and executor pods. + env = env + .with_value( + &CONTAINERDEBUG_LOG_DIRECTORY, + format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), + ) + .with_value( + &STACKABLE_PRE_HOOK, + format!( + "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &" + ), + ); + } if self.requirements().is_some() { env = env.with_value( @@ -2088,6 +2114,29 @@ spec: assert!(!command.contains("spark.jars=")); } + #[test] + fn test_containerdebug_is_enabled_by_default_and_can_be_disabled() { + let default_command = build_command_with_openlineage(MINIMAL_APP, "4.1.2"); + assert!(default_command.contains("containerdebug --output=")); + + let disabled_yaml = indoc! {r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: test + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + containerDebug: + enabled: false + "#}; + let disabled_command = build_command_with_openlineage(disabled_yaml, "4.1.2"); + assert!(!disabled_command.contains("containerdebug")); + } + #[test] fn test_openlineage_appends_to_existing_extra_listeners() { let yaml = indoc! {r#" @@ -2467,6 +2516,8 @@ spec: mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar vectorAggregatorConfigMapName: vector-aggregator-discovery + containerDebug: + enabled: false args: - "--verbose" - "--input" diff --git a/rust/operator-binary/src/crd/template_merger.rs b/rust/operator-binary/src/crd/template_merger.rs index f7389e27..7ae10b12 100644 --- a/rust/operator-binary/src/crd/template_merger.rs +++ b/rust/operator-binary/src/crd/template_merger.rs @@ -58,6 +58,11 @@ pub fn deep_merge(base: &SparkApplication, overlay: &SparkApplication) -> SparkA .vector_aggregator_config_map_name .clone() .or_else(|| base.spec.vector_aggregator_config_map_name.clone()), + container_debug: overlay + .spec + .container_debug + .clone() + .or_else(|| base.spec.container_debug.clone()), lineage: overlay .spec .lineage @@ -826,6 +831,48 @@ mod tests { ); } + #[test] + fn test_deep_merge_container_debug() { + let base = yaml_serde::from_str::(indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: base-app + spec: + mode: cluster + mainApplicationFile: base.py + sparkImage: + productVersion: "3.5.8" + containerDebug: + enabled: true + "#}) + .unwrap(); + + let overlay = yaml_serde::from_str::(indoc! {r#" + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: overlay-app + spec: + mode: cluster + mainApplicationFile: overlay.py + sparkImage: + productVersion: "4.1.0" + containerDebug: + enabled: false + "#}) + .unwrap(); + + let merged = deep_merge(&base, &overlay); + + assert_eq!( + merged.spec.container_debug.map(|config| config.enabled), + Some(false) + ); + } + #[test] fn test_deep_merge_pod_overrides() { let base = yaml_serde::from_str::(indoc! {r#" diff --git a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs index 08c1c114..1b971db9 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs @@ -610,6 +610,66 @@ mod tests { assert_eq!(matching[0].value.as_deref(), Some("/custom/log/dir")); } + #[test] + fn containerdebug_can_be_disabled_for_driver_pods() { + let yaml = indoc! {r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-example + namespace: default + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + containerDebug: + enabled: false + "#}; + let validated = validated_from_yaml(yaml, None); + let driver_config = validated + .spark_application + .driver_config() + .expect("the driver config resolves"); + let env = validated + .spark_application + .env(&None, &None, None) + .expect("the base environment can be built"); + + let template = pod_template( + &validated, + SparkApplicationRole::Driver, + &driver_config, + &[], + &env, + &test_service_account(), + ) + .expect("the driver pod template can be built"); + + let env = template + .spec + .expect("the pod template has a spec") + .containers + .iter() + .find(|container| container.name == "spark") + .expect("the spark container exists") + .env + .as_ref() + .expect("the spark container has env vars"); + + assert!( + !env.iter() + .any(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + ); + let pre_hook = env + .iter() + .find(|env_var| env_var.name == "_STACKABLE_PRE_HOOK") + .and_then(|env_var| env_var.value.as_deref()) + .expect("the Spark environment bootstrap hook remains configured"); + assert!(!pre_hook.contains("containerdebug")); + assert!(pre_hook.contains("spark-env.sh")); + } + /// Runs the real validate step against the given SparkApplication YAML, optionally with a /// dereferenced S3 connection. fn validated_from_yaml(