feat: Collect the containerdebug logs

CHANGELOG.md

@@ -10,6 +10,9 @@ All notable changes to this project will be documented in this file.
   ([#186]).
 - Add support for OpenSearch 3.8.0, deprecate 3.6.0 and remove support for 3.4.0 ([#189]).
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#190]).
+- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to
+  the Vector aggregator. The operator enables the file logs of containerdebug for this, with an
+  hourly rotation that keeps at most two files ([#205]).
 
 ### Changed
 
@@ -43,6 +46,9 @@ All notable changes to this project will be documented in this file.
 - Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs,
   which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources.
   See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#172]).
+- The `containerdebug` process is started again. It was guarded with `command --search`, which is
+  fish syntax and fails in bash, so the guard was always false and containerdebug never ran
+  ([#205]).
 
 [#166]: https://github.com/stackabletech/opensearch-operator/pull/166
 [#171]: https://github.com/stackabletech/opensearch-operator/pull/171
@@ -57,6 +63,7 @@ All notable changes to this project will be documented in this file.
 [#199]: https://github.com/stackabletech/opensearch-operator/pull/199
 [#201]: https://github.com/stackabletech/opensearch-operator/pull/201
 [#202]: https://github.com/stackabletech/opensearch-operator/pull/202
+[#205]: https://github.com/stackabletech/opensearch-operator/pull/205
 
 ## [26.7.0] - 2026-07-21
 

Cargo.toml

@@ -14,6 +14,7 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git",
 
 built = { version = "0.8", features = ["chrono", "git2"] }
 clap = "4.6"
+const_format = "0.2"
 futures = { version = "0.3", features = ["compat"] }
 pretty_assertions = "1.4"
 regex = "1.13"

docs/modules/opensearch/pages/usage-guide/logging.adoc

@@ -31,4 +31,12 @@ spec:
                 level: INFO
 ----
 
+NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well.
+They are labelled `container: containerdebug`, although no such Kubernetes container exists.
+The process runs in the background of the `opensearch` container.
+The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name.
+Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated.
+The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/opensearch` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation.
+A path outside `/stackable/log` also leaves the size-capped log volume.
+
 Further information on how to configure logging, can be found in xref:concepts:logging.adoc[].

rust/operator-binary/Cargo.toml

@@ -19,6 +19,7 @@ test-support = []
 stackable-operator.workspace = true
 
 clap.workspace = true
+const_format.workspace = true
 futures.workspace = true
 regex.workspace = true
 semver.workspace = true

rust/operator-binary/src/controller/build/node_config.rs

@@ -2,6 +2,7 @@
 
 use std::str::FromStr;
 
+use const_format::concatcp;
 use serde_json::json;
 use stackable_operator::{
     builder::pod::container::FieldPathEnvVar,
@@ -144,6 +145,9 @@ const CONFIG_OPTION_TRANSPORT_PUBLISH_HOST: &str = "transport.publish_host";
 const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch";
 
 constant!(ENV_VAR_NAME_DISCOVERY_SEED_HOSTS: EnvVarName = CONFIG_OPTION_DISCOVERY_SEED_HOSTS);
+constant!(ENV_VAR_NAME_FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY");
+constant!(ENV_VAR_NAME_FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES");
+constant!(ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD");
 constant!(ENV_VAR_NAME_HTTP_PUBLISH_HOST: EnvVarName = CONFIG_OPTION_HTTP_PUBLISH_HOST);
 constant!(ENV_VAR_NAME_INITIAL_CLUSTER_MANAGER_NODES: EnvVarName = CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES);
 constant!(ENV_VAR_NAME_NETWORK_PUBLISH_HOST: EnvVarName = CONFIG_OPTION_NETWORK_PUBLISH_HOST);
@@ -381,7 +385,19 @@ impl NodeConfig {
                         .collect::<Vec<_>>(),
                 )
                 .expect("Node roles cannot contain commas, therefore creating a comma-separated list is safe."),
-            );
+            )
+            // Let the `containerdebug` process write its tracing information to a file. These
+            // are the generic `stackable-telemetry` variables and apply to the whole OpenSearch
+            // container, so any other telemetry-based process in it would log into the same
+            // directory and the Vector agent would label its events with
+            // `container: containerdebug`.
+            .with_value(
+                &ENV_VAR_NAME_FILE_LOG_DIRECTORY,
+                concatcp!(STACKABLE_LOG_DIR, "/containerdebug"),
+            )
+            // Rotation is opt-in, so without these the file would grow unbounded.
+            .with_value(&ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD, "hourly")
+            .with_value(&ENV_VAR_NAME_FILE_LOG_MAX_FILES, "2");
 
         if let Some(initial_cluster_manager_nodes) = self.initial_cluster_manager_nodes() {
             env_vars = env_vars.with_value(
@@ -725,6 +741,9 @@ mod tests {
     fn test_constants() {
         // Test that dereferencing the constants does not panic.
         let _ = *ENV_VAR_NAME_DISCOVERY_SEED_HOSTS;
+        let _ = *ENV_VAR_NAME_FILE_LOG_DIRECTORY;
+        let _ = *ENV_VAR_NAME_FILE_LOG_MAX_FILES;
+        let _ = *ENV_VAR_NAME_FILE_LOG_ROTATION_PERIOD;
         let _ = *ENV_VAR_NAME_HTTP_PUBLISH_HOST;
         let _ = *ENV_VAR_NAME_INITIAL_CLUSTER_MANAGER_NODES;
         let _ = *ENV_VAR_NAME_NETWORK_PUBLISH_HOST;
@@ -763,6 +782,15 @@ mod tests {
 
         assert_eq!(
             EnvVarSet::new()
+                .with_value(
+                    &EnvVarName::from_str_unsafe("FILE_LOG_DIRECTORY"),
+                    "/stackable/log/containerdebug",
+                )
+                .with_value(&EnvVarName::from_str_unsafe("FILE_LOG_MAX_FILES"), "2")
+                .with_value(
+                    &EnvVarName::from_str_unsafe("FILE_LOG_ROTATION_PERIOD"),
+                    "hourly",
+                )
                 .with_value(&EnvVarName::from_str_unsafe("TEST"), "value")
                 .with_field_path(
                     &EnvVarName::from_str_unsafe("_POD_NAME"),