feat: Collect the containerdebug logs

CHANGELOG.md

@@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file.
   ([#831]).
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#835]).
 - Name nodes now have a default affinity to the OPA Pods when OPA authorization is configured ([#839]).
+- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to
+  the Vector aggregator ([#857]).
 
 ### Changed
 
@@ -62,6 +64,10 @@ All notable changes to this project will be documented in this file.
   replicas, and never for datanodes ([#830]).
 - Set `dfs.encrypt.data.transfer.cipher.suites` instead of the misspelt `dfs.encrypt.data.transfer.cipher.suite`.
   Hadoop ignored the misspelt key, so block data was never encrypted with AES/CTR and was only protected by SASL DIGEST-MD5 ([#854]).
+- The `containerdebug` process writes its log file again. The operator set
+  `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it
+  moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is
+  rotated hourly and at most two files are kept, because rotation is opt-in ([#857]).
 
 ### Removed
 
@@ -89,6 +95,7 @@ All notable changes to this project will be documented in this file.
 [#848]: https://github.com/stackabletech/hdfs-operator/pull/848
 [#850]: https://github.com/stackabletech/hdfs-operator/pull/850
 [#854]: https://github.com/stackabletech/hdfs-operator/pull/854
+[#857]: https://github.com/stackabletech/hdfs-operator/pull/857
 
 ## [26.7.0] - 2026-07-21
 

docs/modules/hdfs/pages/usage-guide/logging-log-aggregation.adoc

@@ -22,4 +22,12 @@ spec:
         enableVectorAgent: true
 ----
 
+NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well.
+They are labelled `container: containerdebug`, although no such Kubernetes container exists.
+The process runs in the background of the `hdfs` container.
+The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name.
+Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated.
+The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/hdfs` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation.
+A path outside `/stackable/log` also leaves the size-capped log volume.
+
 Further information on how to configure logging, can be found in xref:concepts:logging.adoc[].

rust/operator-binary/src/controller/build/container.rs

@@ -11,6 +11,7 @@
 //!
 use std::{collections::BTreeMap, str::FromStr};
 
+use const_format::concatcp;
 use indoc::formatdoc;
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::{
@@ -115,7 +116,9 @@ const ENV_TOPOLOGY_LABELS: &str = "TOPOLOGY_LABELS";
 const ENV_HADOOP_OPTS: &str = "HADOOP_OPTS";
 const ENV_KRB5_CONFIG: &str = "KRB5_CONFIG";
 const ENV_KRB5_CLIENT_KTNAME: &str = "KRB5_CLIENT_KTNAME";
-const ENV_CONTAINERDEBUG_LOG_DIRECTORY: &str = "CONTAINERDEBUG_LOG_DIRECTORY";
+const ENV_FILE_LOG_DIRECTORY: &str = "FILE_LOG_DIRECTORY";
+const ENV_FILE_LOG_MAX_FILES: &str = "FILE_LOG_MAX_FILES";
+const ENV_FILE_LOG_ROTATION_PERIOD: &str = "FILE_LOG_ROTATION_PERIOD";
 
 type Result<T, E = Error> = std::result::Result<T, E>;
 
@@ -570,6 +573,7 @@ impl ContainerConfig {
                 {COMMON_BASH_TRAP_FUNCTIONS}
                 {remove_vector_shutdown_file_command}
                 prepare_signal_handlers
+                mkdir --parents "$FILE_LOG_DIRECTORY"
                 containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &
                 if [[ -d {LISTENER_VOLUME_DIR} ]]; then
                     export POD_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)
@@ -859,15 +863,34 @@ impl ContainerConfig {
             );
         }
 
-        // Needed for the `containerdebug` process to log it's tracing information to.
-        env.insert(
-            ENV_CONTAINERDEBUG_LOG_DIRECTORY.to_string(),
-            EnvVar {
-                name: ENV_CONTAINERDEBUG_LOG_DIRECTORY.to_string(),
-                value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")),
-                value_from: None,
-            },
-        );
+        // Let the `containerdebug` process write its tracing information to a file. These are the
+        // generic `stackable-telemetry` variables and apply to the whole container, so any other
+        // telemetry-based process in it would log into the same directory and the Vector agent
+        // would label its events with `container: containerdebug`.
+        // Rotation is opt-in, so without the last two the file would grow unbounded.
+        // Only the main container runs `containerdebug`, so the variables would have no effect
+        // anywhere else.
+        if let ContainerConfig::Hdfs { .. } = self {
+            for (name, value) in [
+                // The directory must be exactly one level below the log directory, because the
+                // Vector agent derives the container name from that path segment.
+                (
+                    ENV_FILE_LOG_DIRECTORY,
+                    concatcp!(STACKABLE_LOG_DIR, "/containerdebug"),
+                ),
+                (ENV_FILE_LOG_ROTATION_PERIOD, "hourly"),
+                (ENV_FILE_LOG_MAX_FILES, "2"),
+            ] {
+                env.insert(
+                    name.to_owned(),
+                    EnvVar {
+                        name: name.to_owned(),
+                        value: Some(value.to_owned()),
+                        ..EnvVar::default()
+                    },
+                );
+            }
+        }
 
         // Overrides need to come last
         let mut env_override_vars: BTreeMap<String, EnvVar> = inputs
@@ -973,6 +996,10 @@ impl ContainerConfig {
                 VolumeBuilder::new(ContainerConfig::STACKABLE_LOG_VOLUME_MOUNT_NAME)
                     .empty_dir(EmptyDirVolumeSource {
                         medium: None,
+                        // Only the log files listed here are budgeted. The `containerdebug`
+                        // files in this volume are rotated by time, not by size, so they have
+                        // no budget; raising `FILE_LOG_LEVEL` via `envOverrides` can therefore
+                        // evict the Pod.
                         size_limit: Some(
                             product_logging::framework::calculate_log_volume_size_limit(&[
                                 MAX_HDFS_LOG_FILE_SIZE,

rust/operator-binary/src/controller/build/properties/product_logging/mod.rs

@@ -176,5 +176,7 @@ mod tests {
         assert!(!content.is_empty());
         // HDFS containers log via log4j to `*.log4j.xml`, so the `files_log4j` source matches them.
         assert!(content.contains("files_log4j"));
+        assert!(content.contains("files_tracing_rs"));
+        assert!(content.contains("*.tracing-rs.json"));
     }
 }